2024-11-24 19:54:33,004 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 19:54:33,026 main DEBUG Took 0.019139 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 19:54:33,026 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 19:54:33,026 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 19:54:33,027 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 19:54:33,028 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,034 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 19:54:33,045 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,047 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,047 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,048 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,048 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,048 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,050 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,050 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,050 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,051 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,051 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,052 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,052 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,052 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,053 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,053 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,054 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,054 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,054 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,055 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,055 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,055 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,056 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,056 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:54:33,057 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,057 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 19:54:33,058 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:54:33,060 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 19:54:33,061 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 19:54:33,062 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 19:54:33,063 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 19:54:33,063 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 19:54:33,071 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 19:54:33,073 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 19:54:33,075 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 19:54:33,075 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 19:54:33,076 main DEBUG createAppenders(={Console}) 2024-11-24 19:54:33,076 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-24 19:54:33,077 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 19:54:33,077 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-24 19:54:33,077 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 19:54:33,078 main DEBUG OutputStream closed 2024-11-24 19:54:33,078 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 19:54:33,078 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 19:54:33,078 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-24 19:54:33,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 19:54:33,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 19:54:33,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 19:54:33,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 19:54:33,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 19:54:33,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 19:54:33,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 19:54:33,151 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 19:54:33,151 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 19:54:33,151 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 19:54:33,152 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 19:54:33,152 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 19:54:33,152 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 19:54:33,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 19:54:33,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 19:54:33,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 19:54:33,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 19:54:33,154 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 19:54:33,156 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 19:54:33,156 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-24 19:54:33,157 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 19:54:33,157 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-24T19:54:33,169 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-24 19:54:33,172 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 19:54:33,172 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T19:54:33,477 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b 2024-11-24T19:54:33,500 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=539, ProcessCount=11, AvailableMemoryMB=11613 2024-11-24T19:54:33,586 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T19:54:33,695 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:33,708 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/cluster_9dd76bc7-0d7a-b73c-7eed-92594bf7a784, deleteOnExit=true 2024-11-24T19:54:33,845 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/cluster_9dd76bc7-0d7a-b73c-7eed-92594bf7a784/zookeeper_0, clientPort=63566, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/cluster_9dd76bc7-0d7a-b73c-7eed-92594bf7a784/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/cluster_9dd76bc7-0d7a-b73c-7eed-92594bf7a784/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T19:54:33,858 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63566 2024-11-24T19:54:33,859 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:33,934 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be with version=8 2024-11-24T19:54:33,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/hbase-staging 2024-11-24T19:54:34,023 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T19:54:34,248 INFO [Time-limited test {}] client.ConnectionUtils(128): master/32a03c886245:0 server-side Connection retries=45 2024-11-24T19:54:34,257 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:34,258 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:34,268 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T19:54:34,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:34,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T19:54:34,411 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T19:54:34,492 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T19:54:34,527 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T19:54:34,538 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T19:54:34,543 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T19:54:34,582 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 94668 (auto-detected) 2024-11-24T19:54:34,583 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T19:54:34,604 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34257 2024-11-24T19:54:34,626 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34257 connecting to ZooKeeper ensemble=127.0.0.1:63566 2024-11-24T19:54:34,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:342570x0, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T19:54:34,756 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34257-0x1016fb30ea60000 connected 2024-11-24T19:54:34,831 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:34,833 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:34,840 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:54:34,844 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be, hbase.cluster.distributed=false 2024-11-24T19:54:34,866 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T19:54:34,871 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34257 2024-11-24T19:54:34,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34257 2024-11-24T19:54:34,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34257 2024-11-24T19:54:34,873 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34257 2024-11-24T19:54:34,874 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34257 2024-11-24T19:54:34,951 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32a03c886245:0 server-side Connection retries=45 2024-11-24T19:54:34,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:34,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:34,953 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T19:54:34,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:34,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T19:54:34,956 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T19:54:34,958 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T19:54:34,959 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39037 2024-11-24T19:54:34,961 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39037 connecting to ZooKeeper ensemble=127.0.0.1:63566 2024-11-24T19:54:34,962 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:34,963 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:34,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:390370x0, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T19:54:34,984 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:390370x0, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:54:34,985 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39037-0x1016fb30ea60001 connected 2024-11-24T19:54:34,992 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T19:54:35,004 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/bucket.cache 2024-11-24T19:54:35,009 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T19:54:35,010 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T19:54:35,012 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T19:54:35,014 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T19:54:35,014 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T19:54:35,019 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T19:54:35,024 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T19:54:35,025 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T19:54:35,026 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/bucket.persistence is complete. 2024-11-24T19:54:35,027 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T19:54:35,033 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T19:54:35,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39037 2024-11-24T19:54:35,035 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39037 2024-11-24T19:54:35,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39037 2024-11-24T19:54:35,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39037 2024-11-24T19:54:35,042 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39037 2024-11-24T19:54:35,060 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32a03c886245:0 server-side Connection retries=45 2024-11-24T19:54:35,060 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:35,060 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:35,060 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T19:54:35,061 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:35,061 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T19:54:35,061 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T19:54:35,061 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T19:54:35,062 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35001 2024-11-24T19:54:35,065 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35001 connecting to ZooKeeper ensemble=127.0.0.1:63566 2024-11-24T19:54:35,065 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:35,066 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:35,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:350010x0, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T19:54:35,092 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:350010x0, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:54:35,092 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35001-0x1016fb30ea60002 connected 2024-11-24T19:54:35,093 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T19:54:35,094 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/bucket.cache 2024-11-24T19:54:35,094 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T19:54:35,095 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T19:54:35,095 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T19:54:35,099 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T19:54:35,099 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T19:54:35,099 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T19:54:35,099 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T19:54:35,100 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T19:54:35,100 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/bucket.persistence is complete. 2024-11-24T19:54:35,101 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T19:54:35,103 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T19:54:35,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35001 2024-11-24T19:54:35,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35001 2024-11-24T19:54:35,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35001 2024-11-24T19:54:35,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35001 2024-11-24T19:54:35,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35001 2024-11-24T19:54:35,127 DEBUG [M:0;32a03c886245:34257 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;32a03c886245:34257 2024-11-24T19:54:35,128 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/32a03c886245,34257,1732478074091 2024-11-24T19:54:35,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:54:35,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:54:35,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:54:35,144 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/32a03c886245,34257,1732478074091 2024-11-24T19:54:35,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T19:54:35,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T19:54:35,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,185 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T19:54:35,186 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/32a03c886245,34257,1732478074091 from backup master directory 2024-11-24T19:54:35,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/32a03c886245,34257,1732478074091 2024-11-24T19:54:35,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:54:35,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:54:35,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:54:35,200 WARN [master/32a03c886245:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T19:54:35,200 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=32a03c886245,34257,1732478074091 2024-11-24T19:54:35,202 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T19:54:35,204 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T19:54:35,211 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/hbase.id] with ID: dc814939-dc2f-4d93-ad3f-73a0c53e1e17 2024-11-24T19:54:35,212 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/.tmp/hbase.id 2024-11-24T19:54:35,217 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/hbase.id] 2024-11-24T19:54:35,249 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:35,255 INFO [master/32a03c886245:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T19:54:35,273 INFO [master/32a03c886245:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-24T19:54:35,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,324 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:54:35,327 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T19:54:35,366 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T19:54:35,371 INFO [master/32a03c886245:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:54:35,420 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store 2024-11-24T19:54:35,439 INFO [master/32a03c886245:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T19:54:35,442 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:35,444 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T19:54:35,444 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:35,444 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:35,446 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T19:54:35,446 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:35,446 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:35,447 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732478075444Disabling compacts and flushes for region at 1732478075444Disabling writes for close at 1732478075446 (+2 ms)Writing region close event to WAL at 1732478075446Closed at 1732478075446 2024-11-24T19:54:35,447 WARN [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/.initializing 2024-11-24T19:54:35,447 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/WALs/32a03c886245,34257,1732478074091 2024-11-24T19:54:35,459 INFO [master/32a03c886245:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T19:54:35,472 INFO [master/32a03c886245:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32a03c886245%2C34257%2C1732478074091, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/WALs/32a03c886245,34257,1732478074091, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/oldWALs, maxLogs=10 2024-11-24T19:54:35,506 INFO [master/32a03c886245:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/WALs/32a03c886245,34257,1732478074091/32a03c886245%2C34257%2C1732478074091.1732478075477 2024-11-24T19:54:35,506 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T19:54:35,507 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:54:35,507 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:35,511 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:35,512 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:35,545 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:35,571 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T19:54:35,575 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:35,576 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:35,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:35,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T19:54:35,581 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:35,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:54:35,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:35,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T19:54:35,586 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:35,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:54:35,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:35,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T19:54:35,592 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:35,593 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:54:35,593 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:35,597 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:35,598 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:35,600 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:35,601 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:35,604 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T19:54:35,606 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:35,612 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:54:35,614 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62445987, jitterRate=-0.06948228180408478}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T19:54:35,625 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732478075523Initializing all the Stores at 1732478075524 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478075524Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478075525 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478075525Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478075525Cleaning up temporary data from old regions at 1732478075601 (+76 ms)Region opened successfully at 1732478075624 (+23 ms) 2024-11-24T19:54:35,626 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T19:54:35,667 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d7128c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32a03c886245/172.17.0.2:0 2024-11-24T19:54:35,702 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T19:54:35,717 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T19:54:35,718 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T19:54:35,722 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T19:54:35,723 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T19:54:35,730 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-24T19:54:35,731 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T19:54:35,763 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T19:54:35,775 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T19:54:35,791 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T19:54:35,794 INFO [master/32a03c886245:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T19:54:35,796 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T19:54:35,807 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T19:54:35,810 INFO [master/32a03c886245:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T19:54:35,814 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T19:54:35,824 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T19:54:35,827 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T19:54:35,832 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T19:54:35,854 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T19:54:35,865 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T19:54:35,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T19:54:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T19:54:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T19:54:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,887 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=32a03c886245,34257,1732478074091, sessionid=0x1016fb30ea60000, setting cluster-up flag (Was=false) 2024-11-24T19:54:35,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,941 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T19:54:35,945 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32a03c886245,34257,1732478074091 2024-11-24T19:54:35,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:35,991 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T19:54:35,994 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32a03c886245,34257,1732478074091 2024-11-24T19:54:36,002 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T19:54:36,015 INFO [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(746): ClusterId : dc814939-dc2f-4d93-ad3f-73a0c53e1e17 2024-11-24T19:54:36,015 INFO [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(746): ClusterId : dc814939-dc2f-4d93-ad3f-73a0c53e1e17 2024-11-24T19:54:36,018 DEBUG [RS:0;32a03c886245:39037 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T19:54:36,018 DEBUG [RS:1;32a03c886245:35001 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T19:54:36,034 DEBUG [RS:1;32a03c886245:35001 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T19:54:36,034 DEBUG [RS:0;32a03c886245:39037 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T19:54:36,034 DEBUG [RS:1;32a03c886245:35001 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T19:54:36,034 DEBUG [RS:0;32a03c886245:39037 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T19:54:36,042 DEBUG [RS:0;32a03c886245:39037 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T19:54:36,042 DEBUG [RS:1;32a03c886245:35001 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T19:54:36,043 DEBUG [RS:0;32a03c886245:39037 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@560ccf2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32a03c886245/172.17.0.2:0 2024-11-24T19:54:36,043 DEBUG [RS:1;32a03c886245:35001 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd391e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32a03c886245/172.17.0.2:0 2024-11-24T19:54:36,060 DEBUG [RS:0;32a03c886245:39037 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;32a03c886245:39037 2024-11-24T19:54:36,061 DEBUG [RS:1;32a03c886245:35001 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;32a03c886245:35001 2024-11-24T19:54:36,063 INFO [RS:0;32a03c886245:39037 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T19:54:36,063 INFO [RS:1;32a03c886245:35001 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T19:54:36,063 INFO [RS:0;32a03c886245:39037 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T19:54:36,063 INFO [RS:1;32a03c886245:35001 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T19:54:36,063 DEBUG [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T19:54:36,063 DEBUG [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T19:54:36,065 INFO [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(2659): reportForDuty to master=32a03c886245,34257,1732478074091 with port=39037, startcode=1732478074917 2024-11-24T19:54:36,065 INFO [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(2659): reportForDuty to master=32a03c886245,34257,1732478074091 with port=35001, startcode=1732478075059 2024-11-24T19:54:36,077 DEBUG [RS:0;32a03c886245:39037 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T19:54:36,077 DEBUG [RS:1;32a03c886245:35001 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T19:54:36,082 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T19:54:36,093 INFO [master/32a03c886245:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T19:54:36,100 INFO [master/32a03c886245:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T19:54:36,107 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 32a03c886245,34257,1732478074091 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T19:54:36,115 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/32a03c886245:0, corePoolSize=5, maxPoolSize=5 2024-11-24T19:54:36,116 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/32a03c886245:0, corePoolSize=5, maxPoolSize=5 2024-11-24T19:54:36,116 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/32a03c886245:0, corePoolSize=5, maxPoolSize=5 2024-11-24T19:54:36,116 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/32a03c886245:0, corePoolSize=5, maxPoolSize=5 2024-11-24T19:54:36,116 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/32a03c886245:0, corePoolSize=10, maxPoolSize=10 2024-11-24T19:54:36,116 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,117 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/32a03c886245:0, corePoolSize=2, maxPoolSize=2 2024-11-24T19:54:36,117 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,129 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T19:54:36,130 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T19:54:36,131 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732478106131 2024-11-24T19:54:36,134 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T19:54:36,135 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T19:54:36,135 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:36,135 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T19:54:36,138 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T19:54:36,139 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T19:54:36,139 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T19:54:36,139 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T19:54:36,140 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,147 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T19:54:36,149 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T19:54:36,150 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T19:54:36,152 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38891, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T19:54:36,152 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43545, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T19:54:36,152 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T19:54:36,152 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T19:54:36,154 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/32a03c886245:0:becomeActiveMaster-HFileCleaner.large.0-1732478076153,5,FailOnTimeoutGroup] 2024-11-24T19:54:36,156 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/32a03c886245:0:becomeActiveMaster-HFileCleaner.small.0-1732478076154,5,FailOnTimeoutGroup] 2024-11-24T19:54:36,156 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,157 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T19:54:36,158 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,158 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,160 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34257 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32a03c886245,35001,1732478075059 2024-11-24T19:54:36,162 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34257 {}] master.ServerManager(517): Registering regionserver=32a03c886245,35001,1732478075059 2024-11-24T19:54:36,166 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T19:54:36,166 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be 2024-11-24T19:54:36,171 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34257 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32a03c886245,39037,1732478074917 2024-11-24T19:54:36,171 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34257 {}] master.ServerManager(517): Registering regionserver=32a03c886245,39037,1732478074917 2024-11-24T19:54:36,176 DEBUG [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be 2024-11-24T19:54:36,176 DEBUG [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be 2024-11-24T19:54:36,176 DEBUG [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T19:54:36,176 DEBUG [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T19:54:36,176 DEBUG [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T19:54:36,176 DEBUG [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T19:54:36,176 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:36,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T19:54:36,185 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T19:54:36,185 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:36,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:36,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T19:54:36,190 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T19:54:36,190 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:36,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:36,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T19:54:36,194 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T19:54:36,195 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:36,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:36,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T19:54:36,199 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T19:54:36,199 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:36,199 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:36,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T19:54:36,200 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740 2024-11-24T19:54:36,200 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740 2024-11-24T19:54:36,200 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T19:54:36,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T19:54:36,201 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T19:54:36,202 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T19:54:36,207 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:54:36,208 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65616232, jitterRate=-0.022241950035095215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T19:54:36,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732478076176Initializing all the Stores at 1732478076177 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478076177Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478076178 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478076178Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478076178Cleaning up temporary data from old regions at 1732478076201 (+23 ms)Region opened successfully at 1732478076212 (+11 ms) 2024-11-24T19:54:36,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T19:54:36,213 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T19:54:36,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T19:54:36,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T19:54:36,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T19:54:36,214 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T19:54:36,215 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732478076213Disabling compacts and flushes for region at 1732478076213Disabling writes for close at 1732478076213Writing region close event to WAL at 1732478076214 (+1 ms)Closed at 1732478076214 2024-11-24T19:54:36,216 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T19:54:36,216 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T19:54:36,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T19:54:36,228 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T19:54:36,230 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T19:54:36,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T19:54:36,240 DEBUG [RS:1;32a03c886245:35001 {}] zookeeper.ZKUtil(111): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32a03c886245,35001,1732478075059 2024-11-24T19:54:36,240 DEBUG [RS:0;32a03c886245:39037 {}] zookeeper.ZKUtil(111): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32a03c886245,39037,1732478074917 2024-11-24T19:54:36,240 WARN [RS:1;32a03c886245:35001 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T19:54:36,240 WARN [RS:0;32a03c886245:39037 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T19:54:36,240 INFO [RS:1;32a03c886245:35001 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:54:36,240 INFO [RS:0;32a03c886245:39037 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:54:36,240 DEBUG [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,35001,1732478075059 2024-11-24T19:54:36,240 DEBUG [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,39037,1732478074917 2024-11-24T19:54:36,243 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32a03c886245,39037,1732478074917] 2024-11-24T19:54:36,243 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32a03c886245,35001,1732478075059] 2024-11-24T19:54:36,266 INFO [RS:0;32a03c886245:39037 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T19:54:36,266 INFO [RS:1;32a03c886245:35001 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T19:54:36,283 INFO [RS:1;32a03c886245:35001 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T19:54:36,283 INFO [RS:0;32a03c886245:39037 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T19:54:36,288 INFO [RS:1;32a03c886245:35001 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T19:54:36,288 INFO [RS:0;32a03c886245:39037 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T19:54:36,288 INFO [RS:1;32a03c886245:35001 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,288 INFO [RS:0;32a03c886245:39037 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,289 INFO [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T19:54:36,290 INFO [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T19:54:36,294 INFO [RS:0;32a03c886245:39037 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T19:54:36,294 INFO [RS:1;32a03c886245:35001 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T19:54:36,296 INFO [RS:1;32a03c886245:35001 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,296 INFO [RS:0;32a03c886245:39037 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,296 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,296 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,296 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,296 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,296 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,296 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,297 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,297 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,297 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,297 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,297 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32a03c886245:0, corePoolSize=2, maxPoolSize=2 2024-11-24T19:54:36,297 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32a03c886245:0, corePoolSize=2, maxPoolSize=2 2024-11-24T19:54:36,297 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,297 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,297 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,297 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,298 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,298 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,298 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,298 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,298 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,298 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,298 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,298 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:36,298 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32a03c886245:0, corePoolSize=3, maxPoolSize=3 2024-11-24T19:54:36,298 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32a03c886245:0, corePoolSize=3, maxPoolSize=3 2024-11-24T19:54:36,299 DEBUG [RS:1;32a03c886245:35001 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32a03c886245:0, corePoolSize=3, maxPoolSize=3 2024-11-24T19:54:36,299 DEBUG [RS:0;32a03c886245:39037 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32a03c886245:0, corePoolSize=3, maxPoolSize=3 2024-11-24T19:54:36,300 INFO [RS:0;32a03c886245:39037 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,300 INFO [RS:0;32a03c886245:39037 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,300 INFO [RS:0;32a03c886245:39037 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,300 INFO [RS:0;32a03c886245:39037 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,300 INFO [RS:0;32a03c886245:39037 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,301 INFO [RS:0;32a03c886245:39037 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,39037,1732478074917-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T19:54:36,301 INFO [RS:1;32a03c886245:35001 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,301 INFO [RS:1;32a03c886245:35001 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,301 INFO [RS:1;32a03c886245:35001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,301 INFO [RS:1;32a03c886245:35001 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,301 INFO [RS:1;32a03c886245:35001 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,301 INFO [RS:1;32a03c886245:35001 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,35001,1732478075059-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T19:54:36,321 INFO [RS:0;32a03c886245:39037 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T19:54:36,323 INFO [RS:0;32a03c886245:39037 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,39037,1732478074917-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,324 INFO [RS:0;32a03c886245:39037 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,324 INFO [RS:0;32a03c886245:39037 {}] regionserver.Replication(171): 32a03c886245,39037,1732478074917 started 2024-11-24T19:54:36,326 INFO [RS:1;32a03c886245:35001 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T19:54:36,326 INFO [RS:1;32a03c886245:35001 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,35001,1732478075059-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,327 INFO [RS:1;32a03c886245:35001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,327 INFO [RS:1;32a03c886245:35001 {}] regionserver.Replication(171): 32a03c886245,35001,1732478075059 started 2024-11-24T19:54:36,342 INFO [RS:0;32a03c886245:39037 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,343 INFO [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(1482): Serving as 32a03c886245,39037,1732478074917, RpcServer on 32a03c886245/172.17.0.2:39037, sessionid=0x1016fb30ea60001 2024-11-24T19:54:36,344 DEBUG [RS:0;32a03c886245:39037 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T19:54:36,344 DEBUG [RS:0;32a03c886245:39037 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32a03c886245,39037,1732478074917 2024-11-24T19:54:36,344 DEBUG [RS:0;32a03c886245:39037 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a03c886245,39037,1732478074917' 2024-11-24T19:54:36,344 DEBUG [RS:0;32a03c886245:39037 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T19:54:36,345 DEBUG [RS:0;32a03c886245:39037 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T19:54:36,346 DEBUG [RS:0;32a03c886245:39037 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T19:54:36,346 DEBUG [RS:0;32a03c886245:39037 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T19:54:36,346 DEBUG [RS:0;32a03c886245:39037 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32a03c886245,39037,1732478074917 2024-11-24T19:54:36,346 DEBUG [RS:0;32a03c886245:39037 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a03c886245,39037,1732478074917' 2024-11-24T19:54:36,346 DEBUG [RS:0;32a03c886245:39037 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T19:54:36,347 DEBUG [RS:0;32a03c886245:39037 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T19:54:36,348 DEBUG [RS:0;32a03c886245:39037 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T19:54:36,348 INFO [RS:0;32a03c886245:39037 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T19:54:36,348 INFO [RS:0;32a03c886245:39037 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T19:54:36,350 INFO [RS:1;32a03c886245:35001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:36,351 INFO [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(1482): Serving as 32a03c886245,35001,1732478075059, RpcServer on 32a03c886245/172.17.0.2:35001, sessionid=0x1016fb30ea60002 2024-11-24T19:54:36,351 DEBUG [RS:1;32a03c886245:35001 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T19:54:36,351 DEBUG [RS:1;32a03c886245:35001 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32a03c886245,35001,1732478075059 2024-11-24T19:54:36,351 DEBUG [RS:1;32a03c886245:35001 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a03c886245,35001,1732478075059' 2024-11-24T19:54:36,351 DEBUG [RS:1;32a03c886245:35001 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T19:54:36,352 DEBUG [RS:1;32a03c886245:35001 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T19:54:36,353 DEBUG [RS:1;32a03c886245:35001 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T19:54:36,353 DEBUG [RS:1;32a03c886245:35001 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T19:54:36,353 DEBUG [RS:1;32a03c886245:35001 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32a03c886245,35001,1732478075059 2024-11-24T19:54:36,353 DEBUG [RS:1;32a03c886245:35001 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a03c886245,35001,1732478075059' 2024-11-24T19:54:36,353 DEBUG [RS:1;32a03c886245:35001 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T19:54:36,354 DEBUG [RS:1;32a03c886245:35001 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T19:54:36,355 DEBUG [RS:1;32a03c886245:35001 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T19:54:36,355 INFO [RS:1;32a03c886245:35001 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T19:54:36,355 INFO [RS:1;32a03c886245:35001 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T19:54:36,381 WARN [32a03c886245:34257 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:54:36,455 INFO [RS:0;32a03c886245:39037 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T19:54:36,456 INFO [RS:1;32a03c886245:35001 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T19:54:36,457 INFO [RS:0;32a03c886245:39037 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32a03c886245%2C39037%2C1732478074917, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,39037,1732478074917, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs, maxLogs=55 2024-11-24T19:54:36,458 INFO [RS:1;32a03c886245:35001 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32a03c886245%2C35001%2C1732478075059, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,35001,1732478075059, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs, maxLogs=55 2024-11-24T19:54:36,480 INFO [RS:1;32a03c886245:35001 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,35001,1732478075059/32a03c886245%2C35001%2C1732478075059.1732478076461 2024-11-24T19:54:36,481 DEBUG [RS:1;32a03c886245:35001 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T19:54:36,488 INFO [RS:0;32a03c886245:39037 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,39037,1732478074917/32a03c886245%2C39037%2C1732478074917.1732478076461 2024-11-24T19:54:36,491 DEBUG [RS:0;32a03c886245:39037 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T19:54:36,635 DEBUG [32a03c886245:34257 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T19:54:36,649 DEBUG [32a03c886245:34257 {}] balancer.BalancerClusterState(204): Hosts are {32a03c886245=0} racks are {/default-rack=0} 2024-11-24T19:54:36,652 DEBUG [32a03c886245:34257 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T19:54:36,652 DEBUG [32a03c886245:34257 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T19:54:36,652 DEBUG [32a03c886245:34257 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T19:54:36,652 DEBUG [32a03c886245:34257 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T19:54:36,652 INFO [32a03c886245:34257 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T19:54:36,653 INFO [32a03c886245:34257 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T19:54:36,653 DEBUG [32a03c886245:34257 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T19:54:36,659 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=32a03c886245,39037,1732478074917 2024-11-24T19:54:36,664 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32a03c886245,39037,1732478074917, state=OPENING 2024-11-24T19:54:36,723 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T19:54:36,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:36,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:36,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:36,733 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:36,733 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:36,734 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:36,735 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T19:54:36,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=32a03c886245,39037,1732478074917}] 2024-11-24T19:54:36,912 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T19:54:36,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53103, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T19:54:36,922 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T19:54:36,923 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:54:36,924 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T19:54:36,926 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32a03c886245%2C39037%2C1732478074917.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,39037,1732478074917, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs, maxLogs=55 2024-11-24T19:54:36,946 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,39037,1732478074917/32a03c886245%2C39037%2C1732478074917.meta.1732478076928.meta 2024-11-24T19:54:36,946 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T19:54:36,947 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:54:36,948 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T19:54:36,950 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T19:54:36,954 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T19:54:36,958 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T19:54:36,958 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:36,958 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T19:54:36,958 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T19:54:36,960 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T19:54:36,961 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T19:54:36,961 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:36,961 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:36,962 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T19:54:36,962 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T19:54:36,962 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:36,963 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:36,963 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T19:54:36,963 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T19:54:36,963 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:36,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:36,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T19:54:36,964 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T19:54:36,964 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:36,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:36,965 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T19:54:36,966 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740 2024-11-24T19:54:36,966 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740 2024-11-24T19:54:36,967 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T19:54:36,967 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T19:54:36,967 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T19:54:36,968 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T19:54:36,968 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64976125, jitterRate=-0.03178028762340546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T19:54:36,969 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T19:54:36,970 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732478076959Writing region info on filesystem at 1732478076959Initializing all the Stores at 1732478076959Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478076960 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478076960Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478076960Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478076960Cleaning up temporary data from old regions at 1732478076967 (+7 ms)Running coprocessor post-open hooks at 1732478076969 (+2 ms)Region opened successfully at 1732478076970 (+1 ms) 2024-11-24T19:54:36,975 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732478076902 2024-11-24T19:54:36,985 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T19:54:36,986 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T19:54:36,986 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,39037,1732478074917 2024-11-24T19:54:36,988 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32a03c886245,39037,1732478074917, state=OPEN 2024-11-24T19:54:37,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:54:37,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:54:37,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:54:37,025 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:37,025 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:37,025 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:37,026 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=32a03c886245,39037,1732478074917 2024-11-24T19:54:37,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T19:54:37,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=32a03c886245,39037,1732478074917 in 289 msec 2024-11-24T19:54:37,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T19:54:37,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 811 msec 2024-11-24T19:54:37,037 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T19:54:37,037 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T19:54:37,054 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T19:54:37,055 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a03c886245,39037,1732478074917, seqNum=-1] 2024-11-24T19:54:37,071 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T19:54:37,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41231, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T19:54:37,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0540 sec 2024-11-24T19:54:37,090 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732478077090, completionTime=-1 2024-11-24T19:54:37,093 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T19:54:37,093 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T19:54:37,115 INFO [master/32a03c886245:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T19:54:37,115 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732478137115 2024-11-24T19:54:37,115 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732478197115 2024-11-24T19:54:37,115 INFO [master/32a03c886245:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-11-24T19:54:37,117 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T19:54:37,123 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,34257,1732478074091-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:37,123 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,34257,1732478074091-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:37,123 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,34257,1732478074091-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:37,125 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-32a03c886245:34257, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:37,125 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:37,126 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:37,135 DEBUG [master/32a03c886245:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T19:54:37,153 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.953sec 2024-11-24T19:54:37,155 INFO [master/32a03c886245:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T19:54:37,156 INFO [master/32a03c886245:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T19:54:37,157 INFO [master/32a03c886245:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T19:54:37,157 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T19:54:37,157 INFO [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T19:54:37,158 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,34257,1732478074091-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T19:54:37,158 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,34257,1732478074091-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T19:54:37,160 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T19:54:37,160 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T19:54:37,161 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,34257,1732478074091-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:37,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20a6e58f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T19:54:37,230 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T19:54:37,230 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T19:54:37,234 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a03c886245,34257,-1 for getting cluster id 2024-11-24T19:54:37,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T19:54:37,245 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dc814939-dc2f-4d93-ad3f-73a0c53e1e17' 2024-11-24T19:54:37,247 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T19:54:37,247 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dc814939-dc2f-4d93-ad3f-73a0c53e1e17" 2024-11-24T19:54:37,248 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e89455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T19:54:37,248 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a03c886245,34257,-1] 2024-11-24T19:54:37,250 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T19:54:37,252 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:37,253 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T19:54:37,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3da5686d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T19:54:37,256 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T19:54:37,262 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a03c886245,39037,1732478074917, seqNum=-1] 2024-11-24T19:54:37,263 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T19:54:37,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T19:54:37,293 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=32a03c886245,34257,1732478074091 2024-11-24T19:54:37,297 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T19:54:37,304 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 32a03c886245,34257,1732478074091 2024-11-24T19:54:37,307 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@74762d0c 2024-11-24T19:54:37,308 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T19:54:37,310 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T19:54:37,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34257 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732478077293', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:54:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34257 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732478077293 2024-11-24T19:54:37,324 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732478077293 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T19:54:37,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34257 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732478077293" procId is: 4 2024-11-24T19:54:37,327 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:37,329 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732478077293 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T19:54:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34257 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T19:54:37,340 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732478077293-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5450c46b9d61097c563210ef9808e9fd, NAME => 'testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732478077293', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be 2024-11-24T19:54:37,349 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732478077293-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:37,349 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732478077293-pool-0 {}] regionserver.HRegion(1722): Closing 5450c46b9d61097c563210ef9808e9fd, disabling compactions & flushes 2024-11-24T19:54:37,350 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732478077293-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:37,350 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732478077293-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:37,350 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732478077293-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. after waiting 0 ms 2024-11-24T19:54:37,350 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732478077293-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:37,350 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732478077293-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:37,350 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732478077293-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5450c46b9d61097c563210ef9808e9fd: Waiting for close lock at 1732478077349Disabling compacts and flushes for region at 1732478077349Disabling writes for close at 1732478077350 (+1 ms)Writing region close event to WAL at 1732478077350Closed at 1732478077350 2024-11-24T19:54:37,351 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732478077293 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T19:54:37,356 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732478077351"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732478077351"}]},"ts":"1732478077351"} 2024-11-24T19:54:37,360 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T19:54:37,361 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732478077293 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T19:54:37,363 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732478077293","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478077361"}]},"ts":"1732478077361"} 2024-11-24T19:54:37,367 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732478077293, state=ENABLING in hbase:meta 2024-11-24T19:54:37,367 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {32a03c886245=0} racks are {/default-rack=0} 2024-11-24T19:54:37,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T19:54:37,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T19:54:37,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T19:54:37,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T19:54:37,368 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T19:54:37,368 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T19:54:37,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T19:54:37,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732478077293, region=5450c46b9d61097c563210ef9808e9fd, ASSIGN}] 2024-11-24T19:54:37,371 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732478077293, region=5450c46b9d61097c563210ef9808e9fd, ASSIGN 2024-11-24T19:54:37,372 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732478077293, region=5450c46b9d61097c563210ef9808e9fd, ASSIGN; state=OFFLINE, location=32a03c886245,39037,1732478074917; forceNewPlan=false, retain=false 2024-11-24T19:54:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34257 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T19:54:37,526 INFO [32a03c886245:34257 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T19:54:37,528 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5450c46b9d61097c563210ef9808e9fd, regionState=OPENING, regionLocation=32a03c886245,39037,1732478074917 2024-11-24T19:54:37,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732478077293, region=5450c46b9d61097c563210ef9808e9fd, ASSIGN because future has completed 2024-11-24T19:54:37,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5450c46b9d61097c563210ef9808e9fd, server=32a03c886245,39037,1732478074917}] 2024-11-24T19:54:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34257 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T19:54:37,692 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:37,692 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5450c46b9d61097c563210ef9808e9fd, NAME => 'testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:54:37,693 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732478077293 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:37,693 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:37,693 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:37,693 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:37,694 INFO [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:37,697 INFO [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5450c46b9d61097c563210ef9808e9fd columnFamilyName family 2024-11-24T19:54:37,697 DEBUG [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:37,698 INFO [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] regionserver.HStore(327): Store=5450c46b9d61097c563210ef9808e9fd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:54:37,698 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:37,699 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:37,699 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:37,699 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:37,699 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:37,700 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:37,705 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:54:37,705 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5450c46b9d61097c563210ef9808e9fd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74038349, jitterRate=0.10325737297534943}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:54:37,705 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:37,706 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5450c46b9d61097c563210ef9808e9fd: Running coprocessor pre-open hook at 1732478077693Writing region info on filesystem at 1732478077693Initializing all the Stores at 1732478077694 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478077694Cleaning up temporary data from old regions at 1732478077699 (+5 ms)Running coprocessor post-open hooks at 1732478077705 (+6 ms)Region opened successfully at 1732478077706 (+1 ms) 2024-11-24T19:54:37,707 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd., pid=6, masterSystemTime=1732478077689 2024-11-24T19:54:37,710 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:37,710 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:37,710 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5450c46b9d61097c563210ef9808e9fd, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,39037,1732478074917 2024-11-24T19:54:37,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5450c46b9d61097c563210ef9808e9fd, server=32a03c886245,39037,1732478074917 because future has completed 2024-11-24T19:54:37,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T19:54:37,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5450c46b9d61097c563210ef9808e9fd, server=32a03c886245,39037,1732478074917 in 181 msec 2024-11-24T19:54:37,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T19:54:37,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732478077293, region=5450c46b9d61097c563210ef9808e9fd, ASSIGN in 347 msec 2024-11-24T19:54:37,719 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732478077293 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T19:54:37,720 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732478077293","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478077720"}]},"ts":"1732478077720"} 2024-11-24T19:54:37,722 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732478077293, state=ENABLED in hbase:meta 2024-11-24T19:54:37,723 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732478077293 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T19:54:37,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732478077293 in 405 msec 2024-11-24T19:54:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34257 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T19:54:37,963 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732478077293 get assigned. Timeout = 60000ms 2024-11-24T19:54:37,963 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732478077293 completed 2024-11-24T19:54:37,968 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:54:37,975 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732478077293 assigned to meta. Checking AM states. 2024-11-24T19:54:37,975 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:54:37,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732478077293 assigned. 2024-11-24T19:54:37,985 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732478077293', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd., hostname=32a03c886245,39037,1732478074917, seqNum=2] 2024-11-24T19:54:37,995 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5450c46b9d61097c563210ef9808e9fd 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T19:54:38,017 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d6458597d47c43f79aba0d096e52619e 2024-11-24T19:54:38,053 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/.tmp/family/d6458597d47c43f79aba0d096e52619e is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T19:54:38,057 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/.tmp/family/d6458597d47c43f79aba0d096e52619e 2024-11-24T19:54:38,089 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/.tmp/family/d6458597d47c43f79aba0d096e52619e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/family/d6458597d47c43f79aba0d096e52619e 2024-11-24T19:54:38,090 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/family/d6458597d47c43f79aba0d096e52619e, delay=947 ms 2024-11-24T19:54:38,092 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/family/d6458597d47c43f79aba0d096e52619e, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T19:54:38,098 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 5450c46b9d61097c563210ef9808e9fd in 102ms, sequenceid=6, compaction requested=false 2024-11-24T19:54:38,099 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732478077293' 2024-11-24T19:54:38,102 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5450c46b9d61097c563210ef9808e9fd: 2024-11-24T19:54:39,040 DEBUG [hfile-prefetch-1732478078091 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d6458597d47c43f79aba0d096e52619e_0. 2024-11-24T19:54:39,045 DEBUG [hfile-prefetch-1732478078091 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d6458597d47c43f79aba0d096e52619e_117. 2024-11-24T19:54:39,046 DEBUG [hfile-prefetch-1732478078091 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/family/d6458597d47c43f79aba0d096e52619e, with total blocks 2, and data blocks 1 2024-11-24T19:54:39,046 DEBUG [hfile-prefetch-1732478078091 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T19:54:39,047 DEBUG [hfile-prefetch-1732478078091 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/family/d6458597d47c43f79aba0d096e52619e, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T19:54:39,047 DEBUG [hfile-prefetch-1732478078091 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/family/d6458597d47c43f79aba0d096e52619e. Will sleep 100ms and try the verification again. 2024-11-24T19:54:39,119 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34257 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=5450c46b9d61097c563210ef9808e9fd, source=32a03c886245,39037,1732478074917, destination=32a03c886245,35001,1732478075059, warming up region on 32a03c886245,35001,1732478075059 2024-11-24T19:54:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34257 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T19:54:39,121 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34257 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=5450c46b9d61097c563210ef9808e9fd, source=32a03c886245,39037,1732478074917, destination=32a03c886245,35001,1732478075059, running balancer 2024-11-24T19:54:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34257 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732478077293, region=5450c46b9d61097c563210ef9808e9fd, REOPEN/MOVE 2024-11-24T19:54:39,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52575, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T19:54:39,122 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732478077293, region=5450c46b9d61097c563210ef9808e9fd, REOPEN/MOVE 2024-11-24T19:54:39,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35001 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35001 {}] regionserver.HRegion(7855): Warmup {ENCODED => 5450c46b9d61097c563210ef9808e9fd, NAME => 'testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:54:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35001 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:39,124 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5450c46b9d61097c563210ef9808e9fd, regionState=CLOSING, regionLocation=32a03c886245,39037,1732478074917 2024-11-24T19:54:39,124 INFO [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,125 INFO [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5450c46b9d61097c563210ef9808e9fd columnFamilyName family 2024-11-24T19:54:39,125 DEBUG [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:39,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732478077293, region=5450c46b9d61097c563210ef9808e9fd, REOPEN/MOVE because future has completed 2024-11-24T19:54:39,128 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-24T19:54:39,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5450c46b9d61097c563210ef9808e9fd, server=32a03c886245,39037,1732478074917}] 2024-11-24T19:54:39,129 DEBUG [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/family/d6458597d47c43f79aba0d096e52619e 2024-11-24T19:54:39,129 INFO [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] regionserver.HStore(327): Store=5450c46b9d61097c563210ef9808e9fd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:54:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35001 {}] regionserver.HRegion(1722): Closing 5450c46b9d61097c563210ef9808e9fd, disabling compactions & flushes 2024-11-24T19:54:39,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35001 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35001 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35001 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. after waiting 0 ms 2024-11-24T19:54:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35001 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,131 DEBUG [StoreFileCloser-5450c46b9d61097c563210ef9808e9fd-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/family/d6458597d47c43f79aba0d096e52619e 2024-11-24T19:54:39,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35001 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35001 {}] regionserver.HRegion(1676): Region close journal for 5450c46b9d61097c563210ef9808e9fd: Waiting for close lock at 1732478079130Disabling compacts and flushes for region at 1732478079130Disabling writes for close at 1732478079130Writing region close event to WAL at 1732478079132 (+2 ms)Closed at 1732478079132 2024-11-24T19:54:39,133 WARN [hfile-prefetch-1732478078091 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/family/d6458597d47c43f79aba0d096e52619e, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-24T19:54:39,134 DEBUG [hfile-prefetch-1732478078091 {}] hfile.PrefetchExecutor(121): Prefetch completed for d6458597d47c43f79aba0d096e52619e 2024-11-24T19:54:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34257 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T19:54:39,199 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T19:54:39,199 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T19:54:39,199 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T19:54:39,199 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T19:54:39,216 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T19:54:39,216 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T19:54:39,216 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T19:54:39,216 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T19:54:39,220 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T19:54:39,220 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732478079199 2024-11-24T19:54:39,220 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T19:54:39,223 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-24T19:54:39,223 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732478079216 2024-11-24T19:54:39,223 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T19:54:39,288 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,288 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-24T19:54:39,289 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 5450c46b9d61097c563210ef9808e9fd, disabling compactions & flushes 2024-11-24T19:54:39,289 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,289 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,289 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. after waiting 0 ms 2024-11-24T19:54:39,289 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,292 DEBUG [StoreFileCloser-5450c46b9d61097c563210ef9808e9fd-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file d6458597d47c43f79aba0d096e52619e, starting offset: 0, end offset: 9223372036854775807 2024-11-24T19:54:39,294 DEBUG [StoreFileCloser-5450c46b9d61097c563210ef9808e9fd-family-1 {}] bucket.BucketCache(856): removed key d6458597d47c43f79aba0d096e52619e_0 from back map with offset lock 2094080 in the evict process 2024-11-24T19:54:39,301 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T19:54:39,302 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,302 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 5450c46b9d61097c563210ef9808e9fd: Waiting for close lock at 1732478079289Running coprocessor pre-close hooks at 1732478079289Disabling compacts and flushes for region at 1732478079289Disabling writes for close at 1732478079289Writing region close event to WAL at 1732478079295 (+6 ms)Running coprocessor post-close hooks at 1732478079301 (+6 ms)Closed at 1732478079302 (+1 ms) 2024-11-24T19:54:39,303 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 5450c46b9d61097c563210ef9808e9fd move to 32a03c886245,35001,1732478075059 record at close sequenceid=6 2024-11-24T19:54:39,306 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,307 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5450c46b9d61097c563210ef9808e9fd, regionState=CLOSED 2024-11-24T19:54:39,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5450c46b9d61097c563210ef9808e9fd, server=32a03c886245,39037,1732478074917 because future has completed 2024-11-24T19:54:39,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T19:54:39,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 5450c46b9d61097c563210ef9808e9fd, server=32a03c886245,39037,1732478074917 in 184 msec 2024-11-24T19:54:39,314 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732478077293, region=5450c46b9d61097c563210ef9808e9fd, REOPEN/MOVE; state=CLOSED, location=32a03c886245,35001,1732478075059; forceNewPlan=false, retain=false 2024-11-24T19:54:39,320 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T19:54:39,320 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T19:54:39,320 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T19:54:39,320 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T19:54:39,323 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T19:54:39,323 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732478079320 2024-11-24T19:54:39,323 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T19:54:39,465 INFO [32a03c886245:34257 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T19:54:39,466 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5450c46b9d61097c563210ef9808e9fd, regionState=OPENING, regionLocation=32a03c886245,35001,1732478075059 2024-11-24T19:54:39,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732478077293, region=5450c46b9d61097c563210ef9808e9fd, REOPEN/MOVE because future has completed 2024-11-24T19:54:39,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5450c46b9d61097c563210ef9808e9fd, server=32a03c886245,35001,1732478075059}] 2024-11-24T19:54:39,632 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,632 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 5450c46b9d61097c563210ef9808e9fd, NAME => 'testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:54:39,633 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732478077293 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,633 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:39,633 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,633 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,635 INFO [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,636 INFO [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5450c46b9d61097c563210ef9808e9fd columnFamilyName family 2024-11-24T19:54:39,636 DEBUG [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:39,638 DEBUG [StoreFileOpener-5450c46b9d61097c563210ef9808e9fd-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/family/d6458597d47c43f79aba0d096e52619e, delay=926 ms 2024-11-24T19:54:39,640 DEBUG [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/family/d6458597d47c43f79aba0d096e52619e 2024-11-24T19:54:39,640 INFO [StoreOpener-5450c46b9d61097c563210ef9808e9fd-1 {}] regionserver.HStore(327): Store=5450c46b9d61097c563210ef9808e9fd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:54:39,640 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,641 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,642 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,642 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,642 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,646 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,646 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 5450c46b9d61097c563210ef9808e9fd; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70588601, jitterRate=0.05185212194919586}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:54:39,646 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,647 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 5450c46b9d61097c563210ef9808e9fd: Running coprocessor pre-open hook at 1732478079634Writing region info on filesystem at 1732478079634Initializing all the Stores at 1732478079634Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478079634Cleaning up temporary data from old regions at 1732478079642 (+8 ms)Running coprocessor post-open hooks at 1732478079646 (+4 ms)Region opened successfully at 1732478079647 (+1 ms) 2024-11-24T19:54:39,648 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd., pid=9, masterSystemTime=1732478079626 2024-11-24T19:54:39,650 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,651 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,651 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5450c46b9d61097c563210ef9808e9fd, regionState=OPEN, openSeqNum=10, regionLocation=32a03c886245,35001,1732478075059 2024-11-24T19:54:39,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5450c46b9d61097c563210ef9808e9fd, server=32a03c886245,35001,1732478075059 because future has completed 2024-11-24T19:54:39,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T19:54:39,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 5450c46b9d61097c563210ef9808e9fd, server=32a03c886245,35001,1732478075059 in 184 msec 2024-11-24T19:54:39,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732478077293, region=5450c46b9d61097c563210ef9808e9fd, REOPEN/MOVE in 537 msec 2024-11-24T19:54:39,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T19:54:39,675 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T19:54:39,675 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:54:39,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:39,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:39,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T19:54:39,680 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T19:54:39,680 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=632300009, stopped=false 2024-11-24T19:54:39,680 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=32a03c886245,34257,1732478074091 2024-11-24T19:54:39,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T19:54:39,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T19:54:39,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T19:54:39,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:39,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:39,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:39,707 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T19:54:39,708 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:54:39,708 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:54:39,708 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T19:54:39,708 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:54:39,708 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:54:39,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:39,709 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32a03c886245,39037,1732478074917' ***** 2024-11-24T19:54:39,710 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T19:54:39,710 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32a03c886245,35001,1732478075059' ***** 2024-11-24T19:54:39,710 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T19:54:39,710 INFO [RS:0;32a03c886245:39037 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T19:54:39,710 INFO [RS:1;32a03c886245:35001 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T19:54:39,711 INFO [RS:0;32a03c886245:39037 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/bucket.persistence 2024-11-24T19:54:39,711 INFO [RS:1;32a03c886245:35001 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/bucket.persistence 2024-11-24T19:54:39,711 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T19:54:39,711 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T19:54:39,711 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T19:54:39,711 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T19:54:39,711 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T19:54:39,711 INFO [RS:0;32a03c886245:39037 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T19:54:39,711 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T19:54:39,711 INFO [RS:1;32a03c886245:35001 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T19:54:39,711 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T19:54:39,711 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T19:54:39,721 DEBUG [RS:0;32a03c886245:39037 {}] bucket.BucketCache(1487): Thread RS:0;32a03c886245:39037 started persisting bucket cache to file 2024-11-24T19:54:39,721 DEBUG [RS:1;32a03c886245:35001 {}] bucket.BucketCache(1487): Thread RS:1;32a03c886245:35001 started persisting bucket cache to file 2024-11-24T19:54:39,722 DEBUG [RS:0;32a03c886245:39037 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T19:54:39,722 DEBUG [RS:1;32a03c886245:35001 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T19:54:39,722 DEBUG [RS:0;32a03c886245:39037 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T19:54:39,722 DEBUG [RS:1;32a03c886245:35001 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T19:54:39,724 DEBUG [RS:1;32a03c886245:35001 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T19:54:39,724 DEBUG [RS:0;32a03c886245:39037 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T19:54:39,724 DEBUG [RS:1;32a03c886245:35001 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732478079722 2024-11-24T19:54:39,724 DEBUG [RS:0;32a03c886245:39037 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732478079721 2024-11-24T19:54:39,724 DEBUG [RS:1;32a03c886245:35001 {}] bucket.BucketCache(1509): Thread RS:1;32a03c886245:35001 finished persisting bucket cache to file, renaming 2024-11-24T19:54:39,724 DEBUG [RS:0;32a03c886245:39037 {}] bucket.BucketCache(1509): Thread RS:0;32a03c886245:39037 finished persisting bucket cache to file, renaming 2024-11-24T19:54:39,725 INFO [RS:1;32a03c886245:35001 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T19:54:39,725 INFO [RS:0;32a03c886245:39037 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T19:54:39,725 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T19:54:39,725 INFO [RS:1;32a03c886245:35001 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T19:54:39,725 INFO [RS:0;32a03c886245:39037 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T19:54:39,725 INFO [RS:0;32a03c886245:39037 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T19:54:39,725 INFO [RS:1;32a03c886245:35001 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T19:54:39,725 INFO [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(959): stopping server 32a03c886245,39037,1732478074917 2024-11-24T19:54:39,725 INFO [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(3091): Received CLOSE for 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,725 INFO [RS:0;32a03c886245:39037 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T19:54:39,725 INFO [RS:0;32a03c886245:39037 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;32a03c886245:39037. 2024-11-24T19:54:39,726 DEBUG [RS:0;32a03c886245:39037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:54:39,726 DEBUG [RS:0;32a03c886245:39037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:39,726 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T19:54:39,726 INFO [RS:0;32a03c886245:39037 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T19:54:39,726 INFO [RS:0;32a03c886245:39037 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T19:54:39,726 INFO [RS:0;32a03c886245:39037 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T19:54:39,726 INFO [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(959): stopping server 32a03c886245,35001,1732478075059 2024-11-24T19:54:39,726 INFO [RS:1;32a03c886245:35001 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T19:54:39,726 INFO [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T19:54:39,726 INFO [RS:1;32a03c886245:35001 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;32a03c886245:35001. 2024-11-24T19:54:39,726 DEBUG [RS:1;32a03c886245:35001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:54:39,726 DEBUG [RS:1;32a03c886245:35001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:39,726 INFO [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T19:54:39,726 INFO [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T19:54:39,726 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T19:54:39,726 DEBUG [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(1325): Online Regions={5450c46b9d61097c563210ef9808e9fd=testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd.} 2024-11-24T19:54:39,726 DEBUG [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T19:54:39,726 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T19:54:39,727 DEBUG [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(1351): Waiting on 5450c46b9d61097c563210ef9808e9fd 2024-11-24T19:54:39,727 DEBUG [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T19:54:39,727 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5450c46b9d61097c563210ef9808e9fd, disabling compactions & flushes 2024-11-24T19:54:39,727 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T19:54:39,727 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,727 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T19:54:39,727 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,727 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. after waiting 0 ms 2024-11-24T19:54:39,727 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T19:54:39,727 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,727 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T19:54:39,727 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T19:54:39,727 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-24T19:54:39,728 DEBUG [StoreFileCloser-5450c46b9d61097c563210ef9808e9fd-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/family/d6458597d47c43f79aba0d096e52619e 2024-11-24T19:54:39,731 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnRegionMove1732478077293/5450c46b9d61097c563210ef9808e9fd/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T19:54:39,732 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,732 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5450c46b9d61097c563210ef9808e9fd: Waiting for close lock at 1732478079726Running coprocessor pre-close hooks at 1732478079727 (+1 ms)Disabling compacts and flushes for region at 1732478079727Disabling writes for close at 1732478079727Writing region close event to WAL at 1732478079728 (+1 ms)Running coprocessor post-close hooks at 1732478079732 (+4 ms)Closed at 1732478079732 2024-11-24T19:54:39,732 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd. 2024-11-24T19:54:39,737 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7d53f7f03c754a199630df55ca85bebf 2024-11-24T19:54:39,754 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/info/7d53f7f03c754a199630df55ca85bebf is 205, key is testBlockEvictionOnRegionMove1732478077293,,1732478077312.5450c46b9d61097c563210ef9808e9fd./info:regioninfo/1732478079651/Put/seqid=0 2024-11-24T19:54:39,755 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/info/7d53f7f03c754a199630df55ca85bebf 2024-11-24T19:54:39,763 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3497cfc2f16e40faa72e53b6d89e0678 2024-11-24T19:54:39,775 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/ns/3497cfc2f16e40faa72e53b6d89e0678 is 43, key is default/ns:d/1732478077077/Put/seqid=0 2024-11-24T19:54:39,776 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/ns/3497cfc2f16e40faa72e53b6d89e0678 2024-11-24T19:54:39,784 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b6d80fcf1ecf4a22a6c2a4c7d8fb8c04 2024-11-24T19:54:39,796 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/table/b6d80fcf1ecf4a22a6c2a4c7d8fb8c04 is 78, key is testBlockEvictionOnRegionMove1732478077293/table:state/1732478077720/Put/seqid=0 2024-11-24T19:54:39,797 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/table/b6d80fcf1ecf4a22a6c2a4c7d8fb8c04 2024-11-24T19:54:39,799 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/info/7d53f7f03c754a199630df55ca85bebf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/7d53f7f03c754a199630df55ca85bebf 2024-11-24T19:54:39,800 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/7d53f7f03c754a199630df55ca85bebf, delay=983 ms 2024-11-24T19:54:39,802 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/7d53f7f03c754a199630df55ca85bebf, entries=15, sequenceid=15, filesize=7.6 K 2024-11-24T19:54:39,802 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/ns/3497cfc2f16e40faa72e53b6d89e0678 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/ns/3497cfc2f16e40faa72e53b6d89e0678 2024-11-24T19:54:39,803 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/ns/3497cfc2f16e40faa72e53b6d89e0678, delay=944 ms 2024-11-24T19:54:39,805 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/ns/3497cfc2f16e40faa72e53b6d89e0678, entries=2, sequenceid=15, filesize=5.0 K 2024-11-24T19:54:39,805 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/table/b6d80fcf1ecf4a22a6c2a4c7d8fb8c04 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/table/b6d80fcf1ecf4a22a6c2a4c7d8fb8c04 2024-11-24T19:54:39,806 INFO [regionserver/32a03c886245:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T19:54:39,806 INFO [regionserver/32a03c886245:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T19:54:39,806 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/table/b6d80fcf1ecf4a22a6c2a4c7d8fb8c04, delay=954 ms 2024-11-24T19:54:39,807 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/table/b6d80fcf1ecf4a22a6c2a4c7d8fb8c04, entries=2, sequenceid=15, filesize=5.3 K 2024-11-24T19:54:39,808 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=15, compaction requested=false 2024-11-24T19:54:39,808 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T19:54:39,810 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/7d53f7f03c754a199630df55ca85bebf 2024-11-24T19:54:39,810 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/ns/3497cfc2f16e40faa72e53b6d89e0678 2024-11-24T19:54:39,811 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/table/b6d80fcf1ecf4a22a6c2a4c7d8fb8c04 2024-11-24T19:54:39,815 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-24T19:54:39,816 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T19:54:39,816 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T19:54:39,816 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732478079726Running coprocessor pre-close hooks at 1732478079727 (+1 ms)Disabling compacts and flushes for region at 1732478079727Disabling writes for close at 1732478079727Obtaining lock to block concurrent updates at 1732478079727Preparing flush snapshotting stores in 1588230740 at 1732478079727Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732478079728 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732478079728Flushing 1588230740/info: creating writer at 1732478079728Flushing 1588230740/info: appending metadata at 1732478079753 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732478079753Flushing 1588230740/ns: creating writer at 1732478079757 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732478079774 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732478079775 (+1 ms)Flushing 1588230740/table: creating writer at 1732478079778 (+3 ms)Flushing 1588230740/table: appending metadata at 1732478079796 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732478079796Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fefebd8: reopening flushed file at 1732478079799 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4284bdbb: reopening flushed file at 1732478079802 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13da71e6: reopening flushed file at 1732478079805 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=15, compaction requested=false at 1732478079808 (+3 ms)Writing region close event to WAL at 1732478079811 (+3 ms)Running coprocessor post-close hooks at 1732478079816 (+5 ms)Closed at 1732478079816 2024-11-24T19:54:39,817 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T19:54:39,927 INFO [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(976): stopping server 32a03c886245,35001,1732478075059; all regions closed. 2024-11-24T19:54:39,927 INFO [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(976): stopping server 32a03c886245,39037,1732478074917; all regions closed. 2024-11-24T19:54:39,934 DEBUG [RS:1;32a03c886245:35001 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs 2024-11-24T19:54:39,934 DEBUG [RS:0;32a03c886245:39037 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs 2024-11-24T19:54:39,934 INFO [RS:1;32a03c886245:35001 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32a03c886245%2C35001%2C1732478075059:(num 1732478076461) 2024-11-24T19:54:39,934 INFO [RS:0;32a03c886245:39037 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32a03c886245%2C39037%2C1732478074917.meta:.meta(num 1732478076928) 2024-11-24T19:54:39,935 DEBUG [RS:1;32a03c886245:35001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:39,935 INFO [RS:1;32a03c886245:35001 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T19:54:39,935 INFO [RS:1;32a03c886245:35001 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T19:54:39,935 INFO [RS:1;32a03c886245:35001 {}] hbase.ChoreService(370): Chore service for: regionserver/32a03c886245:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T19:54:39,936 INFO [RS:1;32a03c886245:35001 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T19:54:39,936 INFO [RS:1;32a03c886245:35001 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T19:54:39,936 INFO [regionserver/32a03c886245:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T19:54:39,936 INFO [RS:1;32a03c886245:35001 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T19:54:39,936 INFO [RS:1;32a03c886245:35001 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T19:54:39,937 INFO [RS:1;32a03c886245:35001 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35001 2024-11-24T19:54:39,939 DEBUG [RS:0;32a03c886245:39037 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs 2024-11-24T19:54:39,939 INFO [RS:0;32a03c886245:39037 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32a03c886245%2C39037%2C1732478074917:(num 1732478076461) 2024-11-24T19:54:39,939 DEBUG [RS:0;32a03c886245:39037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:39,939 INFO [RS:0;32a03c886245:39037 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T19:54:39,939 INFO [RS:0;32a03c886245:39037 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T19:54:39,939 INFO [RS:0;32a03c886245:39037 {}] hbase.ChoreService(370): Chore service for: regionserver/32a03c886245:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T19:54:39,940 INFO [RS:0;32a03c886245:39037 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T19:54:39,940 INFO [regionserver/32a03c886245:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T19:54:39,940 INFO [RS:0;32a03c886245:39037 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39037 2024-11-24T19:54:39,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T19:54:39,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32a03c886245,35001,1732478075059 2024-11-24T19:54:39,981 INFO [RS:1;32a03c886245:35001 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T19:54:40,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32a03c886245,39037,1732478074917 2024-11-24T19:54:40,040 INFO [RS:0;32a03c886245:39037 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T19:54:40,049 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32a03c886245,39037,1732478074917] 2024-11-24T19:54:40,065 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32a03c886245,39037,1732478074917 already deleted, retry=false 2024-11-24T19:54:40,066 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 32a03c886245,39037,1732478074917 expired; onlineServers=1 2024-11-24T19:54:40,066 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32a03c886245,35001,1732478075059] 2024-11-24T19:54:40,073 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32a03c886245,35001,1732478075059 already deleted, retry=false 2024-11-24T19:54:40,074 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 32a03c886245,35001,1732478075059 expired; onlineServers=0 2024-11-24T19:54:40,074 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '32a03c886245,34257,1732478074091' ***** 2024-11-24T19:54:40,074 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T19:54:40,075 INFO [M:0;32a03c886245:34257 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T19:54:40,075 INFO [M:0;32a03c886245:34257 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T19:54:40,075 DEBUG [M:0;32a03c886245:34257 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T19:54:40,076 DEBUG [M:0;32a03c886245:34257 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T19:54:40,076 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T19:54:40,076 DEBUG [master/32a03c886245:0:becomeActiveMaster-HFileCleaner.small.0-1732478076154 {}] cleaner.HFileCleaner(306): Exit Thread[master/32a03c886245:0:becomeActiveMaster-HFileCleaner.small.0-1732478076154,5,FailOnTimeoutGroup] 2024-11-24T19:54:40,076 DEBUG [master/32a03c886245:0:becomeActiveMaster-HFileCleaner.large.0-1732478076153 {}] cleaner.HFileCleaner(306): Exit Thread[master/32a03c886245:0:becomeActiveMaster-HFileCleaner.large.0-1732478076153,5,FailOnTimeoutGroup] 2024-11-24T19:54:40,077 INFO [M:0;32a03c886245:34257 {}] hbase.ChoreService(370): Chore service for: master/32a03c886245:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T19:54:40,077 INFO [M:0;32a03c886245:34257 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T19:54:40,077 DEBUG [M:0;32a03c886245:34257 {}] master.HMaster(1795): Stopping service threads 2024-11-24T19:54:40,077 INFO [M:0;32a03c886245:34257 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T19:54:40,077 INFO [M:0;32a03c886245:34257 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T19:54:40,078 INFO [M:0;32a03c886245:34257 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T19:54:40,078 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T19:54:40,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T19:54:40,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,082 DEBUG [M:0;32a03c886245:34257 {}] zookeeper.ZKUtil(347): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T19:54:40,082 WARN [M:0;32a03c886245:34257 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T19:54:40,082 INFO [M:0;32a03c886245:34257 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/.lastflushedseqids 2024-11-24T19:54:40,090 INFO [M:0;32a03c886245:34257 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T19:54:40,091 INFO [M:0;32a03c886245:34257 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T19:54:40,091 DEBUG [M:0;32a03c886245:34257 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T19:54:40,091 INFO [M:0;32a03c886245:34257 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:40,091 DEBUG [M:0;32a03c886245:34257 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:40,091 DEBUG [M:0;32a03c886245:34257 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T19:54:40,091 DEBUG [M:0;32a03c886245:34257 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:40,091 INFO [M:0;32a03c886245:34257 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-24T19:54:40,099 DEBUG [M:0;32a03c886245:34257 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on adc8ea9904f34398843eda3b17d91266 2024-11-24T19:54:40,114 DEBUG [M:0;32a03c886245:34257 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/adc8ea9904f34398843eda3b17d91266 is 82, key is hbase:meta,,1/info:regioninfo/1732478076986/Put/seqid=0 2024-11-24T19:54:40,115 INFO [M:0;32a03c886245:34257 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/adc8ea9904f34398843eda3b17d91266 2024-11-24T19:54:40,123 DEBUG [M:0;32a03c886245:34257 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9480e2969998456bb1ad98130bfbee6d 2024-11-24T19:54:40,136 DEBUG [M:0;32a03c886245:34257 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9480e2969998456bb1ad98130bfbee6d is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732478077725/Put/seqid=0 2024-11-24T19:54:40,137 INFO [M:0;32a03c886245:34257 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9480e2969998456bb1ad98130bfbee6d 2024-11-24T19:54:40,145 DEBUG [M:0;32a03c886245:34257 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c060028b84ba4baa8e08ab96ebf9591c 2024-11-24T19:54:40,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:40,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35001-0x1016fb30ea60002, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:40,149 INFO [RS:1;32a03c886245:35001 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T19:54:40,150 INFO [RS:1;32a03c886245:35001 {}] regionserver.HRegionServer(1031): Exiting; stopping=32a03c886245,35001,1732478075059; zookeeper connection closed. 2024-11-24T19:54:40,150 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@19102d7c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@19102d7c 2024-11-24T19:54:40,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:40,157 INFO [RS:0;32a03c886245:39037 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T19:54:40,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39037-0x1016fb30ea60001, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:40,157 INFO [RS:0;32a03c886245:39037 {}] regionserver.HRegionServer(1031): Exiting; stopping=32a03c886245,39037,1732478074917; zookeeper connection closed. 2024-11-24T19:54:40,157 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4a2a3ec5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4a2a3ec5 2024-11-24T19:54:40,158 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-24T19:54:40,159 DEBUG [M:0;32a03c886245:34257 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c060028b84ba4baa8e08ab96ebf9591c is 69, key is 32a03c886245,35001,1732478075059/rs:state/1732478076164/Put/seqid=0 2024-11-24T19:54:40,160 INFO [M:0;32a03c886245:34257 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c060028b84ba4baa8e08ab96ebf9591c 2024-11-24T19:54:40,162 DEBUG [M:0;32a03c886245:34257 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/adc8ea9904f34398843eda3b17d91266 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/adc8ea9904f34398843eda3b17d91266 2024-11-24T19:54:40,164 INFO [M:0;32a03c886245:34257 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/adc8ea9904f34398843eda3b17d91266, entries=8, sequenceid=84, filesize=5.5 K 2024-11-24T19:54:40,164 DEBUG [M:0;32a03c886245:34257 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9480e2969998456bb1ad98130bfbee6d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9480e2969998456bb1ad98130bfbee6d 2024-11-24T19:54:40,166 INFO [M:0;32a03c886245:34257 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9480e2969998456bb1ad98130bfbee6d, entries=9, sequenceid=84, filesize=6.5 K 2024-11-24T19:54:40,167 DEBUG [M:0;32a03c886245:34257 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c060028b84ba4baa8e08ab96ebf9591c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c060028b84ba4baa8e08ab96ebf9591c 2024-11-24T19:54:40,169 INFO [M:0;32a03c886245:34257 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c060028b84ba4baa8e08ab96ebf9591c, entries=2, sequenceid=84, filesize=5.1 K 2024-11-24T19:54:40,170 INFO [M:0;32a03c886245:34257 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=84, compaction requested=false 2024-11-24T19:54:40,172 INFO [M:0;32a03c886245:34257 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:40,172 DEBUG [M:0;32a03c886245:34257 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732478080091Disabling compacts and flushes for region at 1732478080091Disabling writes for close at 1732478080091Obtaining lock to block concurrent updates at 1732478080091Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732478080091Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1732478080092 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732478080092Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732478080092Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732478080114 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732478080114Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732478080117 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732478080135 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732478080135Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732478080139 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732478080158 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732478080158Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ae17a3d: reopening flushed file at 1732478080161 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68dc4653: reopening flushed file at 1732478080164 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cf993d9: reopening flushed file at 1732478080166 (+2 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=84, compaction requested=false at 1732478080170 (+4 ms)Writing region close event to WAL at 1732478080171 (+1 ms)Closed at 1732478080171 2024-11-24T19:54:40,173 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T19:54:40,173 INFO [M:0;32a03c886245:34257 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T19:54:40,174 INFO [M:0;32a03c886245:34257 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34257 2024-11-24T19:54:40,174 INFO [M:0;32a03c886245:34257 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T19:54:40,290 INFO [M:0;32a03c886245:34257 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T19:54:40,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:40,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34257-0x1016fb30ea60000, quorum=127.0.0.1:63566, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:40,301 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T19:54:40,321 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T19:54:40,321 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:40,329 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732478079801 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732478079638 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/32a03c886245:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/32a03c886245:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1732478078091 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732478079803 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/32a03c886245:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/32a03c886245:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=576 (was 539) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11229 (was 11613) 2024-11-24T19:54:40,334 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=576, ProcessCount=11, AvailableMemoryMB=11228 2024-11-24T19:54:40,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/hadoop-log-dir so I do NOT create it in target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3 2024-11-24T19:54:40,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/hadoop-tmp-dir so I do NOT create it in target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3 2024-11-24T19:54:40,335 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:40,337 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/cluster_d617d811-1fb6-8e66-575d-d5275a09f11d, deleteOnExit=true 2024-11-24T19:54:40,340 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/cluster_d617d811-1fb6-8e66-575d-d5275a09f11d/zookeeper_0, clientPort=54759, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/cluster_d617d811-1fb6-8e66-575d-d5275a09f11d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/cluster_d617d811-1fb6-8e66-575d-d5275a09f11d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T19:54:40,341 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54759 2024-11-24T19:54:40,352 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be with version=8 2024-11-24T19:54:40,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/hbase-staging 2024-11-24T19:54:40,355 INFO [Time-limited test {}] client.ConnectionUtils(128): master/32a03c886245:0 server-side Connection retries=45 2024-11-24T19:54:40,355 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:40,356 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:40,356 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T19:54:40,356 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:40,356 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T19:54:40,356 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T19:54:40,356 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T19:54:40,358 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41101 2024-11-24T19:54:40,360 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41101 connecting to ZooKeeper ensemble=127.0.0.1:54759 2024-11-24T19:54:40,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:411010x0, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T19:54:40,404 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41101-0x1016fb328190000 connected 2024-11-24T19:54:40,466 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:40,466 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:40,468 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:54:40,468 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be, hbase.cluster.distributed=false 2024-11-24T19:54:40,471 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T19:54:40,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41101 2024-11-24T19:54:40,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41101 2024-11-24T19:54:40,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41101 2024-11-24T19:54:40,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41101 2024-11-24T19:54:40,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41101 2024-11-24T19:54:40,490 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32a03c886245:0 server-side Connection retries=45 2024-11-24T19:54:40,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:40,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:40,490 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T19:54:40,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:40,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T19:54:40,490 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T19:54:40,490 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T19:54:40,491 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33775 2024-11-24T19:54:40,492 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33775 connecting to ZooKeeper ensemble=127.0.0.1:54759 2024-11-24T19:54:40,493 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:40,493 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:40,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:337750x0, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T19:54:40,499 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33775-0x1016fb328190001 connected 2024-11-24T19:54:40,499 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:54:40,499 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T19:54:40,500 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.cache 2024-11-24T19:54:40,500 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T19:54:40,500 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T19:54:40,500 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T19:54:40,501 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T19:54:40,501 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T19:54:40,501 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T19:54:40,501 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T19:54:40,501 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T19:54:40,502 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.persistence is complete. 2024-11-24T19:54:40,502 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T19:54:40,503 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T19:54:40,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33775 2024-11-24T19:54:40,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33775 2024-11-24T19:54:40,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33775 2024-11-24T19:54:40,505 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33775 2024-11-24T19:54:40,505 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33775 2024-11-24T19:54:40,521 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32a03c886245:0 server-side Connection retries=45 2024-11-24T19:54:40,521 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:40,521 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:40,521 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T19:54:40,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:40,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T19:54:40,522 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T19:54:40,522 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T19:54:40,523 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38051 2024-11-24T19:54:40,525 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38051 connecting to ZooKeeper ensemble=127.0.0.1:54759 2024-11-24T19:54:40,525 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:40,525 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:40,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380510x0, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T19:54:40,533 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:54:40,533 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38051-0x1016fb328190002 connected 2024-11-24T19:54:40,533 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T19:54:40,534 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.cache 2024-11-24T19:54:40,534 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T19:54:40,534 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T19:54:40,534 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T19:54:40,534 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T19:54:40,534 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T19:54:40,534 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T19:54:40,535 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T19:54:40,535 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T19:54:40,535 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.persistence is complete. 2024-11-24T19:54:40,536 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T19:54:40,537 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T19:54:40,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38051 2024-11-24T19:54:40,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38051 2024-11-24T19:54:40,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38051 2024-11-24T19:54:40,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38051 2024-11-24T19:54:40,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38051 2024-11-24T19:54:40,550 DEBUG [M:0;32a03c886245:41101 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;32a03c886245:41101 2024-11-24T19:54:40,550 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/32a03c886245,41101,1732478080355 2024-11-24T19:54:40,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:54:40,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:54:40,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:54:40,557 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/32a03c886245,41101,1732478080355 2024-11-24T19:54:40,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T19:54:40,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T19:54:40,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,566 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T19:54:40,566 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/32a03c886245,41101,1732478080355 from backup master directory 2024-11-24T19:54:40,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/32a03c886245,41101,1732478080355 2024-11-24T19:54:40,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:54:40,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:54:40,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:54:40,573 WARN [master/32a03c886245:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T19:54:40,574 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=32a03c886245,41101,1732478080355 2024-11-24T19:54:40,575 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/hbase.id] with ID: c5ef859f-2a7e-4fa3-96ae-e7c0023a9b60 2024-11-24T19:54:40,575 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/.tmp/hbase.id 2024-11-24T19:54:40,578 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/hbase.id] 2024-11-24T19:54:40,613 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:40,613 INFO [master/32a03c886245:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T19:54:40,618 INFO [master/32a03c886245:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 5ms. 2024-11-24T19:54:40,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,627 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:54:40,628 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T19:54:40,628 INFO [master/32a03c886245:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:54:40,645 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store 2024-11-24T19:54:40,653 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:40,653 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T19:54:40,653 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:40,653 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:40,653 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T19:54:40,653 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:40,653 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:40,653 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732478080653Disabling compacts and flushes for region at 1732478080653Disabling writes for close at 1732478080653Writing region close event to WAL at 1732478080653Closed at 1732478080653 2024-11-24T19:54:40,654 WARN [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/.initializing 2024-11-24T19:54:40,654 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/WALs/32a03c886245,41101,1732478080355 2024-11-24T19:54:40,661 INFO [master/32a03c886245:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T19:54:40,665 INFO [master/32a03c886245:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32a03c886245%2C41101%2C1732478080355, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/WALs/32a03c886245,41101,1732478080355, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/oldWALs, maxLogs=10 2024-11-24T19:54:40,683 INFO [master/32a03c886245:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/WALs/32a03c886245,41101,1732478080355/32a03c886245%2C41101%2C1732478080355.1732478080665 2024-11-24T19:54:40,683 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T19:54:40,683 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:54:40,683 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:40,683 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:40,683 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:40,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:40,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T19:54:40,688 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:40,689 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:40,689 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:40,692 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T19:54:40,692 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:40,692 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:54:40,692 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:40,696 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T19:54:40,696 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:40,696 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:54:40,696 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:40,699 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T19:54:40,699 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:40,699 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:54:40,700 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:40,701 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:40,701 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:40,701 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:40,701 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:40,701 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T19:54:40,702 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:54:40,708 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:54:40,708 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65935045, jitterRate=-0.017491266131401062}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T19:54:40,709 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732478080684Initializing all the Stores at 1732478080684Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478080684Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478080684Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478080684Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478080684Cleaning up temporary data from old regions at 1732478080701 (+17 ms)Region opened successfully at 1732478080709 (+8 ms) 2024-11-24T19:54:40,710 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T19:54:40,714 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@668e9d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32a03c886245/172.17.0.2:0 2024-11-24T19:54:40,715 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T19:54:40,715 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T19:54:40,715 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T19:54:40,716 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T19:54:40,716 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T19:54:40,717 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-24T19:54:40,717 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T19:54:40,719 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T19:54:40,721 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T19:54:40,740 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T19:54:40,741 INFO [master/32a03c886245:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T19:54:40,742 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T19:54:40,798 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T19:54:40,799 INFO [master/32a03c886245:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T19:54:40,800 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T19:54:40,815 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T19:54:40,817 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T19:54:40,823 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T19:54:40,826 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T19:54:40,832 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T19:54:40,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T19:54:40,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T19:54:40,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T19:54:40,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,841 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=32a03c886245,41101,1732478080355, sessionid=0x1016fb328190000, setting cluster-up flag (Was=false) 2024-11-24T19:54:40,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,882 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T19:54:40,884 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32a03c886245,41101,1732478080355 2024-11-24T19:54:40,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:40,923 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T19:54:40,925 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32a03c886245,41101,1732478080355 2024-11-24T19:54:40,926 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T19:54:40,927 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T19:54:40,928 INFO [master/32a03c886245:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T19:54:40,928 INFO [master/32a03c886245:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T19:54:40,928 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 32a03c886245,41101,1732478080355 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T19:54:40,930 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/32a03c886245:0, corePoolSize=5, maxPoolSize=5 2024-11-24T19:54:40,930 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/32a03c886245:0, corePoolSize=5, maxPoolSize=5 2024-11-24T19:54:40,930 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/32a03c886245:0, corePoolSize=5, maxPoolSize=5 2024-11-24T19:54:40,931 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/32a03c886245:0, corePoolSize=5, maxPoolSize=5 2024-11-24T19:54:40,931 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/32a03c886245:0, corePoolSize=10, maxPoolSize=10 2024-11-24T19:54:40,931 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:40,931 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/32a03c886245:0, corePoolSize=2, maxPoolSize=2 2024-11-24T19:54:40,931 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:40,935 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T19:54:40,936 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T19:54:40,936 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:40,937 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T19:54:40,938 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732478110938 2024-11-24T19:54:40,938 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T19:54:40,938 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T19:54:40,938 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T19:54:40,938 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T19:54:40,938 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T19:54:40,938 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T19:54:40,940 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:40,942 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T19:54:40,942 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T19:54:40,942 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T19:54:40,943 INFO [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(746): ClusterId : c5ef859f-2a7e-4fa3-96ae-e7c0023a9b60 2024-11-24T19:54:40,943 DEBUG [RS:0;32a03c886245:33775 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T19:54:40,943 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T19:54:40,943 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T19:54:40,943 INFO [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(746): ClusterId : c5ef859f-2a7e-4fa3-96ae-e7c0023a9b60 2024-11-24T19:54:40,943 DEBUG [RS:1;32a03c886245:38051 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T19:54:40,943 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/32a03c886245:0:becomeActiveMaster-HFileCleaner.large.0-1732478080943,5,FailOnTimeoutGroup] 2024-11-24T19:54:40,944 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/32a03c886245:0:becomeActiveMaster-HFileCleaner.small.0-1732478080943,5,FailOnTimeoutGroup] 2024-11-24T19:54:40,944 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:40,944 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T19:54:40,944 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:40,944 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:40,949 DEBUG [RS:0;32a03c886245:33775 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T19:54:40,949 DEBUG [RS:1;32a03c886245:38051 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T19:54:40,950 DEBUG [RS:1;32a03c886245:38051 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T19:54:40,950 DEBUG [RS:0;32a03c886245:33775 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T19:54:40,958 DEBUG [RS:1;32a03c886245:38051 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T19:54:40,958 DEBUG [RS:0;32a03c886245:33775 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T19:54:40,958 DEBUG [RS:1;32a03c886245:38051 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65fc57dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32a03c886245/172.17.0.2:0 2024-11-24T19:54:40,959 DEBUG [RS:0;32a03c886245:33775 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@400b6244, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32a03c886245/172.17.0.2:0 2024-11-24T19:54:40,967 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T19:54:40,967 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be 2024-11-24T19:54:40,971 DEBUG [RS:1;32a03c886245:38051 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;32a03c886245:38051 2024-11-24T19:54:40,972 INFO [RS:1;32a03c886245:38051 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T19:54:40,972 INFO [RS:1;32a03c886245:38051 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T19:54:40,972 DEBUG [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T19:54:40,973 INFO [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(2659): reportForDuty to master=32a03c886245,41101,1732478080355 with port=38051, startcode=1732478080521 2024-11-24T19:54:40,973 DEBUG [RS:1;32a03c886245:38051 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T19:54:40,974 DEBUG [RS:0;32a03c886245:33775 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;32a03c886245:33775 2024-11-24T19:54:40,974 INFO [RS:0;32a03c886245:33775 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T19:54:40,974 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:40,974 INFO [RS:0;32a03c886245:33775 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T19:54:40,974 DEBUG [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T19:54:40,975 INFO [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(2659): reportForDuty to master=32a03c886245,41101,1732478080355 with port=33775, startcode=1732478080489 2024-11-24T19:54:40,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T19:54:40,976 DEBUG [RS:0;32a03c886245:33775 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T19:54:40,976 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39069, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T19:54:40,977 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41101 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32a03c886245,38051,1732478080521 2024-11-24T19:54:40,977 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41101 {}] master.ServerManager(517): Registering regionserver=32a03c886245,38051,1732478080521 2024-11-24T19:54:40,978 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53019, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T19:54:40,978 DEBUG [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be 2024-11-24T19:54:40,978 DEBUG [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T19:54:40,978 DEBUG [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T19:54:40,979 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41101 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32a03c886245,33775,1732478080489 2024-11-24T19:54:40,979 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41101 {}] master.ServerManager(517): Registering regionserver=32a03c886245,33775,1732478080489 2024-11-24T19:54:40,980 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T19:54:40,980 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:40,981 DEBUG [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be 2024-11-24T19:54:40,981 DEBUG [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T19:54:40,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:40,981 DEBUG [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T19:54:40,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T19:54:40,985 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T19:54:40,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:40,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:40,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T19:54:40,989 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T19:54:40,989 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:40,989 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:40,989 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T19:54:40,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T19:54:40,993 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T19:54:40,993 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:40,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:40,993 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T19:54:40,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740 2024-11-24T19:54:40,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740 2024-11-24T19:54:40,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T19:54:40,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T19:54:40,995 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T19:54:40,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T19:54:41,000 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:54:41,001 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60645930, jitterRate=-0.09630522131919861}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T19:54:41,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732478080975Initializing all the Stores at 1732478080975Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478080975Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478080975Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478080975Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478080976 (+1 ms)Cleaning up temporary data from old regions at 1732478080994 (+18 ms)Region opened successfully at 1732478081001 (+7 ms) 2024-11-24T19:54:41,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T19:54:41,002 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T19:54:41,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T19:54:41,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T19:54:41,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T19:54:41,002 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T19:54:41,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732478081002Disabling compacts and flushes for region at 1732478081002Disabling writes for close at 1732478081002Writing region close event to WAL at 1732478081002Closed at 1732478081002 2024-11-24T19:54:41,004 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T19:54:41,004 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T19:54:41,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T19:54:41,005 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T19:54:41,005 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T19:54:41,020 DEBUG [RS:1;32a03c886245:38051 {}] zookeeper.ZKUtil(111): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32a03c886245,38051,1732478080521 2024-11-24T19:54:41,020 WARN [RS:1;32a03c886245:38051 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T19:54:41,020 INFO [RS:1;32a03c886245:38051 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:54:41,020 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32a03c886245,38051,1732478080521] 2024-11-24T19:54:41,020 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32a03c886245,33775,1732478080489] 2024-11-24T19:54:41,021 DEBUG [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,38051,1732478080521 2024-11-24T19:54:41,021 DEBUG [RS:0;32a03c886245:33775 {}] zookeeper.ZKUtil(111): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32a03c886245,33775,1732478080489 2024-11-24T19:54:41,021 WARN [RS:0;32a03c886245:33775 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T19:54:41,021 INFO [RS:0;32a03c886245:33775 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:54:41,021 DEBUG [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,33775,1732478080489 2024-11-24T19:54:41,026 INFO [RS:0;32a03c886245:33775 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T19:54:41,026 INFO [RS:1;32a03c886245:38051 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T19:54:41,029 INFO [RS:1;32a03c886245:38051 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T19:54:41,029 INFO [RS:1;32a03c886245:38051 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T19:54:41,030 INFO [RS:1;32a03c886245:38051 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,030 INFO [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T19:54:41,031 INFO [RS:1;32a03c886245:38051 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T19:54:41,031 INFO [RS:1;32a03c886245:38051 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,031 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,032 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,032 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,032 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,032 INFO [RS:0;32a03c886245:33775 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T19:54:41,032 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,032 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32a03c886245:0, corePoolSize=2, maxPoolSize=2 2024-11-24T19:54:41,033 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,033 INFO [RS:0;32a03c886245:33775 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T19:54:41,033 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,033 INFO [RS:0;32a03c886245:33775 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,033 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,033 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,033 INFO [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T19:54:41,033 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,034 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,034 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32a03c886245:0, corePoolSize=3, maxPoolSize=3 2024-11-24T19:54:41,034 DEBUG [RS:1;32a03c886245:38051 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32a03c886245:0, corePoolSize=3, maxPoolSize=3 2024-11-24T19:54:41,034 INFO [RS:0;32a03c886245:33775 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T19:54:41,034 INFO [RS:0;32a03c886245:33775 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,034 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,034 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,035 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,035 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,035 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,035 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32a03c886245:0, corePoolSize=2, maxPoolSize=2 2024-11-24T19:54:41,035 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,035 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,035 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,035 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,035 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,035 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:41,035 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32a03c886245:0, corePoolSize=3, maxPoolSize=3 2024-11-24T19:54:41,035 DEBUG [RS:0;32a03c886245:33775 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32a03c886245:0, corePoolSize=3, maxPoolSize=3 2024-11-24T19:54:41,036 INFO [RS:1;32a03c886245:38051 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,036 INFO [RS:1;32a03c886245:38051 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,036 INFO [RS:1;32a03c886245:38051 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,036 INFO [RS:0;32a03c886245:33775 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,036 INFO [RS:1;32a03c886245:38051 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,036 INFO [RS:0;32a03c886245:33775 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,036 INFO [RS:1;32a03c886245:38051 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,036 INFO [RS:0;32a03c886245:33775 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,036 INFO [RS:1;32a03c886245:38051 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,38051,1732478080521-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T19:54:41,036 INFO [RS:0;32a03c886245:33775 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,036 INFO [RS:0;32a03c886245:33775 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,036 INFO [RS:0;32a03c886245:33775 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,33775,1732478080489-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T19:54:41,050 INFO [RS:0;32a03c886245:33775 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T19:54:41,050 INFO [RS:1;32a03c886245:38051 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T19:54:41,050 INFO [RS:1;32a03c886245:38051 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,38051,1732478080521-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,050 INFO [RS:0;32a03c886245:33775 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,33775,1732478080489-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,051 INFO [RS:1;32a03c886245:38051 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,051 INFO [RS:1;32a03c886245:38051 {}] regionserver.Replication(171): 32a03c886245,38051,1732478080521 started 2024-11-24T19:54:41,051 INFO [RS:0;32a03c886245:33775 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,051 INFO [RS:0;32a03c886245:33775 {}] regionserver.Replication(171): 32a03c886245,33775,1732478080489 started 2024-11-24T19:54:41,064 INFO [RS:1;32a03c886245:38051 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,064 INFO [RS:0;32a03c886245:33775 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,064 INFO [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(1482): Serving as 32a03c886245,33775,1732478080489, RpcServer on 32a03c886245/172.17.0.2:33775, sessionid=0x1016fb328190001 2024-11-24T19:54:41,064 INFO [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(1482): Serving as 32a03c886245,38051,1732478080521, RpcServer on 32a03c886245/172.17.0.2:38051, sessionid=0x1016fb328190002 2024-11-24T19:54:41,064 DEBUG [RS:0;32a03c886245:33775 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T19:54:41,064 DEBUG [RS:1;32a03c886245:38051 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T19:54:41,064 DEBUG [RS:0;32a03c886245:33775 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32a03c886245,33775,1732478080489 2024-11-24T19:54:41,064 DEBUG [RS:1;32a03c886245:38051 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32a03c886245,38051,1732478080521 2024-11-24T19:54:41,064 DEBUG [RS:0;32a03c886245:33775 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a03c886245,33775,1732478080489' 2024-11-24T19:54:41,064 DEBUG [RS:1;32a03c886245:38051 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a03c886245,38051,1732478080521' 2024-11-24T19:54:41,064 DEBUG [RS:1;32a03c886245:38051 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T19:54:41,064 DEBUG [RS:0;32a03c886245:33775 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T19:54:41,065 DEBUG [RS:1;32a03c886245:38051 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T19:54:41,065 DEBUG [RS:0;32a03c886245:33775 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T19:54:41,066 DEBUG [RS:1;32a03c886245:38051 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T19:54:41,066 DEBUG [RS:0;32a03c886245:33775 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T19:54:41,066 DEBUG [RS:1;32a03c886245:38051 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T19:54:41,066 DEBUG [RS:0;32a03c886245:33775 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T19:54:41,066 DEBUG [RS:1;32a03c886245:38051 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32a03c886245,38051,1732478080521 2024-11-24T19:54:41,066 DEBUG [RS:0;32a03c886245:33775 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32a03c886245,33775,1732478080489 2024-11-24T19:54:41,066 DEBUG [RS:1;32a03c886245:38051 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a03c886245,38051,1732478080521' 2024-11-24T19:54:41,066 DEBUG [RS:0;32a03c886245:33775 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a03c886245,33775,1732478080489' 2024-11-24T19:54:41,066 DEBUG [RS:1;32a03c886245:38051 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T19:54:41,066 DEBUG [RS:0;32a03c886245:33775 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T19:54:41,066 DEBUG [RS:1;32a03c886245:38051 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T19:54:41,066 DEBUG [RS:0;32a03c886245:33775 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T19:54:41,067 DEBUG [RS:1;32a03c886245:38051 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T19:54:41,067 INFO [RS:1;32a03c886245:38051 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T19:54:41,067 DEBUG [RS:0;32a03c886245:33775 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T19:54:41,067 INFO [RS:1;32a03c886245:38051 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T19:54:41,067 INFO [RS:0;32a03c886245:33775 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T19:54:41,067 INFO [RS:0;32a03c886245:33775 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T19:54:41,156 WARN [32a03c886245:41101 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:54:41,168 INFO [RS:1;32a03c886245:38051 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T19:54:41,169 INFO [RS:0;32a03c886245:33775 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T19:54:41,170 INFO [RS:1;32a03c886245:38051 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32a03c886245%2C38051%2C1732478080521, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,38051,1732478080521, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs, maxLogs=55 2024-11-24T19:54:41,171 INFO [RS:0;32a03c886245:33775 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32a03c886245%2C33775%2C1732478080489, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,33775,1732478080489, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs, maxLogs=55 2024-11-24T19:54:41,191 INFO [RS:1;32a03c886245:38051 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,38051,1732478080521/32a03c886245%2C38051%2C1732478080521.1732478081176 2024-11-24T19:54:41,191 INFO [RS:0;32a03c886245:33775 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,33775,1732478080489/32a03c886245%2C33775%2C1732478080489.1732478081176 2024-11-24T19:54:41,192 DEBUG [RS:1;32a03c886245:38051 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T19:54:41,192 DEBUG [RS:0;32a03c886245:33775 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T19:54:41,406 DEBUG [32a03c886245:41101 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T19:54:41,407 DEBUG [32a03c886245:41101 {}] balancer.BalancerClusterState(204): Hosts are {32a03c886245=0} racks are {/default-rack=0} 2024-11-24T19:54:41,407 DEBUG [32a03c886245:41101 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T19:54:41,407 DEBUG [32a03c886245:41101 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T19:54:41,407 DEBUG [32a03c886245:41101 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T19:54:41,407 DEBUG [32a03c886245:41101 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T19:54:41,407 INFO [32a03c886245:41101 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T19:54:41,407 INFO [32a03c886245:41101 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T19:54:41,407 DEBUG [32a03c886245:41101 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T19:54:41,408 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=32a03c886245,33775,1732478080489 2024-11-24T19:54:41,409 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32a03c886245,33775,1732478080489, state=OPENING 2024-11-24T19:54:41,448 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T19:54:41,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:41,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:41,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:41,457 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T19:54:41,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:41,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:41,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:41,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=32a03c886245,33775,1732478080489}] 2024-11-24T19:54:41,610 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T19:54:41,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53689, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T19:54:41,614 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T19:54:41,614 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:54:41,615 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T19:54:41,616 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32a03c886245%2C33775%2C1732478080489.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,33775,1732478080489, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs, maxLogs=55 2024-11-24T19:54:41,638 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,33775,1732478080489/32a03c886245%2C33775%2C1732478080489.meta.1732478081617.meta 2024-11-24T19:54:41,639 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T19:54:41,639 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:54:41,640 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T19:54:41,640 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T19:54:41,640 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T19:54:41,640 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T19:54:41,640 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:41,640 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T19:54:41,640 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T19:54:41,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T19:54:41,642 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T19:54:41,642 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:41,642 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:41,642 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T19:54:41,643 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T19:54:41,643 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:41,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:41,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T19:54:41,643 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T19:54:41,643 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:41,644 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:41,644 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T19:54:41,644 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T19:54:41,644 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:41,644 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:41,644 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T19:54:41,645 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740 2024-11-24T19:54:41,646 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740 2024-11-24T19:54:41,646 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T19:54:41,646 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T19:54:41,646 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T19:54:41,647 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T19:54:41,648 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69977874, jitterRate=0.042751580476760864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T19:54:41,648 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T19:54:41,649 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732478081641Writing region info on filesystem at 1732478081641Initializing all the Stores at 1732478081641Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478081641Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478081641Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478081641Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478081641Cleaning up temporary data from old regions at 1732478081646 (+5 ms)Running coprocessor post-open hooks at 1732478081648 (+2 ms)Region opened successfully at 1732478081649 (+1 ms) 2024-11-24T19:54:41,649 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732478081609 2024-11-24T19:54:41,652 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T19:54:41,652 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T19:54:41,653 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,33775,1732478080489 2024-11-24T19:54:41,653 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32a03c886245,33775,1732478080489, state=OPEN 2024-11-24T19:54:41,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:54:41,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:54:41,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:54:41,674 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=32a03c886245,33775,1732478080489 2024-11-24T19:54:41,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:41,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:41,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:41,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T19:54:41,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=32a03c886245,33775,1732478080489 in 217 msec 2024-11-24T19:54:41,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T19:54:41,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 673 msec 2024-11-24T19:54:41,680 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T19:54:41,680 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T19:54:41,684 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T19:54:41,684 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a03c886245,33775,1732478080489, seqNum=-1] 2024-11-24T19:54:41,684 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T19:54:41,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49345, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T19:54:41,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 766 msec 2024-11-24T19:54:41,694 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732478081694, completionTime=-1 2024-11-24T19:54:41,694 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T19:54:41,695 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T19:54:41,698 INFO [master/32a03c886245:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T19:54:41,698 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732478141698 2024-11-24T19:54:41,698 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732478201698 2024-11-24T19:54:41,698 INFO [master/32a03c886245:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-24T19:54:41,699 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,41101,1732478080355-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,699 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,41101,1732478080355-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,699 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,41101,1732478080355-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,699 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-32a03c886245:41101, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,699 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,699 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,701 DEBUG [master/32a03c886245:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T19:54:41,703 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.129sec 2024-11-24T19:54:41,704 INFO [master/32a03c886245:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T19:54:41,704 INFO [master/32a03c886245:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T19:54:41,704 INFO [master/32a03c886245:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T19:54:41,704 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T19:54:41,704 INFO [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T19:54:41,704 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,41101,1732478080355-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T19:54:41,704 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,41101,1732478080355-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T19:54:41,705 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T19:54:41,705 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T19:54:41,705 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,41101,1732478080355-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:41,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d38e96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T19:54:41,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a03c886245,41101,-1 for getting cluster id 2024-11-24T19:54:41,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T19:54:41,745 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c5ef859f-2a7e-4fa3-96ae-e7c0023a9b60' 2024-11-24T19:54:41,746 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T19:54:41,746 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c5ef859f-2a7e-4fa3-96ae-e7c0023a9b60" 2024-11-24T19:54:41,747 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37db8a65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T19:54:41,747 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a03c886245,41101,-1] 2024-11-24T19:54:41,747 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T19:54:41,748 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:41,750 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T19:54:41,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517d04f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T19:54:41,753 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T19:54:41,755 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a03c886245,33775,1732478080489, seqNum=-1] 2024-11-24T19:54:41,756 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T19:54:41,758 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T19:54:41,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=32a03c886245,41101,1732478080355 2024-11-24T19:54:41,761 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T19:54:41,762 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 32a03c886245,41101,1732478080355 2024-11-24T19:54:41,763 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78eb0515 2024-11-24T19:54:41,763 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T19:54:41,765 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T19:54:41,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41101 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732478081761', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:54:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41101 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732478081761 2024-11-24T19:54:41,767 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732478081761 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T19:54:41,767 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:41,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41101 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732478081761" procId is: 4 2024-11-24T19:54:41,768 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732478081761 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T19:54:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41101 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T19:54:41,777 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732478081761-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f43868f9acd93b0e48aedd321a42eb23, NAME => 'testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732478081761', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be 2024-11-24T19:54:41,783 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732478081761-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:41,783 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732478081761-pool-0 {}] regionserver.HRegion(1722): Closing f43868f9acd93b0e48aedd321a42eb23, disabling compactions & flushes 2024-11-24T19:54:41,783 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732478081761-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:41,783 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732478081761-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:41,784 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732478081761-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. after waiting 0 ms 2024-11-24T19:54:41,784 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732478081761-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:41,784 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732478081761-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:41,784 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732478081761-pool-0 {}] regionserver.HRegion(1676): Region close journal for f43868f9acd93b0e48aedd321a42eb23: Waiting for close lock at 1732478081783Disabling compacts and flushes for region at 1732478081783Disabling writes for close at 1732478081784 (+1 ms)Writing region close event to WAL at 1732478081784Closed at 1732478081784 2024-11-24T19:54:41,785 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732478081761 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T19:54:41,785 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732478081785"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732478081785"}]},"ts":"1732478081785"} 2024-11-24T19:54:41,788 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T19:54:41,788 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732478081761 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T19:54:41,789 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732478081761","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478081789"}]},"ts":"1732478081789"} 2024-11-24T19:54:41,791 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732478081761, state=ENABLING in hbase:meta 2024-11-24T19:54:41,792 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {32a03c886245=0} racks are {/default-rack=0} 2024-11-24T19:54:41,792 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T19:54:41,792 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T19:54:41,792 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T19:54:41,792 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T19:54:41,792 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T19:54:41,792 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T19:54:41,792 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T19:54:41,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732478081761, region=f43868f9acd93b0e48aedd321a42eb23, ASSIGN}] 2024-11-24T19:54:41,794 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732478081761, region=f43868f9acd93b0e48aedd321a42eb23, ASSIGN 2024-11-24T19:54:41,795 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732478081761, region=f43868f9acd93b0e48aedd321a42eb23, ASSIGN; state=OFFLINE, location=32a03c886245,33775,1732478080489; forceNewPlan=false, retain=false 2024-11-24T19:54:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41101 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T19:54:41,945 INFO [32a03c886245:41101 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T19:54:41,946 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f43868f9acd93b0e48aedd321a42eb23, regionState=OPENING, regionLocation=32a03c886245,33775,1732478080489 2024-11-24T19:54:41,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732478081761, region=f43868f9acd93b0e48aedd321a42eb23, ASSIGN because future has completed 2024-11-24T19:54:41,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f43868f9acd93b0e48aedd321a42eb23, server=32a03c886245,33775,1732478080489}] 2024-11-24T19:54:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41101 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T19:54:42,111 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:42,111 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f43868f9acd93b0e48aedd321a42eb23, NAME => 'testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:54:42,112 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732478081761 f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:42,112 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:42,112 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:42,112 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:42,113 INFO [StoreOpener-f43868f9acd93b0e48aedd321a42eb23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:42,117 INFO [StoreOpener-f43868f9acd93b0e48aedd321a42eb23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f43868f9acd93b0e48aedd321a42eb23 columnFamilyName family 2024-11-24T19:54:42,117 DEBUG [StoreOpener-f43868f9acd93b0e48aedd321a42eb23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:42,118 INFO [StoreOpener-f43868f9acd93b0e48aedd321a42eb23-1 {}] regionserver.HStore(327): Store=f43868f9acd93b0e48aedd321a42eb23/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:54:42,118 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:42,119 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:42,119 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:42,119 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:42,120 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:42,121 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:42,127 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:54:42,127 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f43868f9acd93b0e48aedd321a42eb23; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70685000, jitterRate=0.05328857898712158}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:54:42,127 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:42,128 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f43868f9acd93b0e48aedd321a42eb23: Running coprocessor pre-open hook at 1732478082112Writing region info on filesystem at 1732478082112Initializing all the Stores at 1732478082113 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478082113Cleaning up temporary data from old regions at 1732478082120 (+7 ms)Running coprocessor post-open hooks at 1732478082127 (+7 ms)Region opened successfully at 1732478082128 (+1 ms) 2024-11-24T19:54:42,129 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23., pid=6, masterSystemTime=1732478082106 2024-11-24T19:54:42,131 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:42,131 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:42,132 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f43868f9acd93b0e48aedd321a42eb23, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,33775,1732478080489 2024-11-24T19:54:42,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f43868f9acd93b0e48aedd321a42eb23, server=32a03c886245,33775,1732478080489 because future has completed 2024-11-24T19:54:42,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T19:54:42,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f43868f9acd93b0e48aedd321a42eb23, server=32a03c886245,33775,1732478080489 in 184 msec 2024-11-24T19:54:42,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T19:54:42,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732478081761, region=f43868f9acd93b0e48aedd321a42eb23, ASSIGN in 345 msec 2024-11-24T19:54:42,140 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732478081761 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T19:54:42,140 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732478081761","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478082140"}]},"ts":"1732478082140"} 2024-11-24T19:54:42,142 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732478081761, state=ENABLED in hbase:meta 2024-11-24T19:54:42,143 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732478081761 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T19:54:42,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732478081761 in 377 msec 2024-11-24T19:54:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41101 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T19:54:42,401 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732478081761 completed 2024-11-24T19:54:42,401 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732478081761 get assigned. Timeout = 60000ms 2024-11-24T19:54:42,401 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:54:42,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732478081761 assigned to meta. Checking AM states. 2024-11-24T19:54:42,406 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:54:42,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732478081761 assigned. 2024-11-24T19:54:42,411 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732478081761', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23., hostname=32a03c886245,33775,1732478080489, seqNum=2] 2024-11-24T19:54:42,418 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f43868f9acd93b0e48aedd321a42eb23 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T19:54:42,429 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8523cab502bc413aa2c43a35f5daef84 2024-11-24T19:54:42,445 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/.tmp/family/8523cab502bc413aa2c43a35f5daef84 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T19:54:42,446 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/.tmp/family/8523cab502bc413aa2c43a35f5daef84 2024-11-24T19:54:42,450 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/.tmp/family/8523cab502bc413aa2c43a35f5daef84 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/family/8523cab502bc413aa2c43a35f5daef84 2024-11-24T19:54:42,451 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/family/8523cab502bc413aa2c43a35f5daef84, delay=922 ms 2024-11-24T19:54:42,453 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/family/8523cab502bc413aa2c43a35f5daef84, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T19:54:42,453 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for f43868f9acd93b0e48aedd321a42eb23 in 35ms, sequenceid=6, compaction requested=false 2024-11-24T19:54:42,453 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732478081761' 2024-11-24T19:54:42,454 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f43868f9acd93b0e48aedd321a42eb23: 2024-11-24T19:54:42,460 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T19:54:43,374 DEBUG [hfile-prefetch-1732478079638 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8523cab502bc413aa2c43a35f5daef84_0. 2024-11-24T19:54:43,375 DEBUG [hfile-prefetch-1732478079638 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8523cab502bc413aa2c43a35f5daef84_117. 2024-11-24T19:54:43,375 DEBUG [hfile-prefetch-1732478079638 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/family/8523cab502bc413aa2c43a35f5daef84, with total blocks 2, and data blocks 1 2024-11-24T19:54:43,375 DEBUG [hfile-prefetch-1732478079638 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T19:54:43,375 DEBUG [hfile-prefetch-1732478079638 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/family/8523cab502bc413aa2c43a35f5daef84, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T19:54:43,375 DEBUG [hfile-prefetch-1732478079638 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/family/8523cab502bc413aa2c43a35f5daef84. Will sleep 100ms and try the verification again. 2024-11-24T19:54:43,455 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;32a03c886245:33775,5,FailOnTimeoutGroup] 2024-11-24T19:54:43,455 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32a03c886245,33775,1732478080489' ***** 2024-11-24T19:54:43,456 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-24T19:54:43,456 INFO [RS:0;32a03c886245:33775 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T19:54:43,466 INFO [RS:0;32a03c886245:33775 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.persistence 2024-11-24T19:54:43,466 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T19:54:43,466 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T19:54:43,466 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T19:54:43,466 INFO [RS:0;32a03c886245:33775 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T19:54:43,466 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T19:54:43,475 DEBUG [hfile-prefetch-1732478079638 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/family/8523cab502bc413aa2c43a35f5daef84, with total blocks 2, and data blocks 1 2024-11-24T19:54:43,476 DEBUG [hfile-prefetch-1732478079638 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-24T19:54:43,476 DEBUG [hfile-prefetch-1732478079638 {}] bucket.BucketCache(2293): found block for file 8523cab502bc413aa2c43a35f5daef84 in the backing map. Acquiring read lock for offset 0 2024-11-24T19:54:43,476 DEBUG [hfile-prefetch-1732478079638 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/family/8523cab502bc413aa2c43a35f5daef84 has now been fully cached. 2024-11-24T19:54:43,476 DEBUG [hfile-prefetch-1732478079638 {}] hfile.PrefetchExecutor(121): Prefetch completed for 8523cab502bc413aa2c43a35f5daef84 2024-11-24T19:54:43,476 DEBUG [RS:0;32a03c886245:33775 {}] bucket.BucketCache(1487): Thread RS:0;32a03c886245:33775 started persisting bucket cache to file 2024-11-24T19:54:43,477 DEBUG [RS:0;32a03c886245:33775 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T19:54:43,477 DEBUG [RS:0;32a03c886245:33775 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-24T19:54:43,481 DEBUG [RS:0;32a03c886245:33775 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-24T19:54:43,481 DEBUG [RS:0;32a03c886245:33775 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732478083476 2024-11-24T19:54:43,481 DEBUG [RS:0;32a03c886245:33775 {}] bucket.BucketCache(1509): Thread RS:0;32a03c886245:33775 finished persisting bucket cache to file, renaming 2024-11-24T19:54:43,482 INFO [RS:0;32a03c886245:33775 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T19:54:43,482 INFO [RS:0;32a03c886245:33775 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T19:54:43,482 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T19:54:43,482 INFO [RS:0;32a03c886245:33775 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T19:54:43,482 INFO [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(3091): Received CLOSE for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:43,482 INFO [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(959): stopping server 32a03c886245,33775,1732478080489 2024-11-24T19:54:43,482 INFO [RS:0;32a03c886245:33775 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T19:54:43,482 INFO [RS:0;32a03c886245:33775 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;32a03c886245:33775. 2024-11-24T19:54:43,482 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T19:54:43,483 DEBUG [RS:0;32a03c886245:33775 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:54:43,483 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f43868f9acd93b0e48aedd321a42eb23, disabling compactions & flushes 2024-11-24T19:54:43,483 DEBUG [RS:0;32a03c886245:33775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:43,483 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:43,483 INFO [RS:0;32a03c886245:33775 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T19:54:43,483 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:43,483 INFO [RS:0;32a03c886245:33775 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T19:54:43,483 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. after waiting 0 ms 2024-11-24T19:54:43,483 INFO [RS:0;32a03c886245:33775 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T19:54:43,483 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:43,483 INFO [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T19:54:43,483 INFO [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T19:54:43,484 DEBUG [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, f43868f9acd93b0e48aedd321a42eb23=testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23.} 2024-11-24T19:54:43,484 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T19:54:43,484 DEBUG [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:43,484 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T19:54:43,484 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T19:54:43,484 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T19:54:43,484 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T19:54:43,484 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T19:54:43,484 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-24T19:54:43,489 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T19:54:43,490 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:43,490 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f43868f9acd93b0e48aedd321a42eb23: Waiting for close lock at 1732478083482Running coprocessor pre-close hooks at 1732478083483 (+1 ms)Disabling compacts and flushes for region at 1732478083483Disabling writes for close at 1732478083483Writing region close event to WAL at 1732478083484 (+1 ms)Running coprocessor post-close hooks at 1732478083489 (+5 ms)Closed at 1732478083490 (+1 ms) 2024-11-24T19:54:43,490 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:43,495 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ef5de49693234ef2a6f22b64958f54d7 2024-11-24T19:54:43,511 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/info/ef5de49693234ef2a6f22b64958f54d7 is 209, key is testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23./info:regioninfo/1732478082132/Put/seqid=0 2024-11-24T19:54:43,511 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/info/ef5de49693234ef2a6f22b64958f54d7 2024-11-24T19:54:43,518 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5d425863b25a41a9b1a7e80eca05a7c8 2024-11-24T19:54:43,532 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/ns/5d425863b25a41a9b1a7e80eca05a7c8 is 43, key is default/ns:d/1732478081688/Put/seqid=0 2024-11-24T19:54:43,533 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/ns/5d425863b25a41a9b1a7e80eca05a7c8 2024-11-24T19:54:43,537 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T19:54:43,537 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T19:54:43,537 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T19:54:43,537 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T19:54:43,539 INFO [regionserver/32a03c886245:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T19:54:43,539 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T19:54:43,539 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732478083537 2024-11-24T19:54:43,539 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T19:54:43,540 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fd954b578a2f44489c6c891a97ac767f 2024-11-24T19:54:43,551 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/table/fd954b578a2f44489c6c891a97ac767f is 80, key is testBlockEvictionOnGracefulStop1732478081761/table:state/1732478082140/Put/seqid=0 2024-11-24T19:54:43,552 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/table/fd954b578a2f44489c6c891a97ac767f 2024-11-24T19:54:43,554 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/info/ef5de49693234ef2a6f22b64958f54d7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/ef5de49693234ef2a6f22b64958f54d7 2024-11-24T19:54:43,555 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/ef5de49693234ef2a6f22b64958f54d7, delay=962 ms 2024-11-24T19:54:43,556 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/ef5de49693234ef2a6f22b64958f54d7, entries=10, sequenceid=11, filesize=7.0 K 2024-11-24T19:54:43,557 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/ns/5d425863b25a41a9b1a7e80eca05a7c8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/ns/5d425863b25a41a9b1a7e80eca05a7c8 2024-11-24T19:54:43,558 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/ns/5d425863b25a41a9b1a7e80eca05a7c8, delay=974 ms 2024-11-24T19:54:43,559 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/ns/5d425863b25a41a9b1a7e80eca05a7c8, entries=2, sequenceid=11, filesize=5.0 K 2024-11-24T19:54:43,559 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/table/fd954b578a2f44489c6c891a97ac767f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/table/fd954b578a2f44489c6c891a97ac767f 2024-11-24T19:54:43,560 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/table/fd954b578a2f44489c6c891a97ac767f, delay=934 ms 2024-11-24T19:54:43,561 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/table/fd954b578a2f44489c6c891a97ac767f, entries=2, sequenceid=11, filesize=5.3 K 2024-11-24T19:54:43,562 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=11, compaction requested=false 2024-11-24T19:54:43,563 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/ef5de49693234ef2a6f22b64958f54d7 2024-11-24T19:54:43,563 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/ns/5d425863b25a41a9b1a7e80eca05a7c8 2024-11-24T19:54:43,564 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/table/fd954b578a2f44489c6c891a97ac767f 2024-11-24T19:54:43,567 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-24T19:54:43,568 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T19:54:43,568 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T19:54:43,568 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732478083484Running coprocessor pre-close hooks at 1732478083484Disabling compacts and flushes for region at 1732478083484Disabling writes for close at 1732478083484Obtaining lock to block concurrent updates at 1732478083484Preparing flush snapshotting stores in 1588230740 at 1732478083484Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732478083485 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732478083485Flushing 1588230740/info: creating writer at 1732478083485Flushing 1588230740/info: appending metadata at 1732478083510 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732478083510Flushing 1588230740/ns: creating writer at 1732478083513 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732478083531 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732478083531Flushing 1588230740/table: creating writer at 1732478083535 (+4 ms)Flushing 1588230740/table: appending metadata at 1732478083550 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732478083551 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56d33d6: reopening flushed file at 1732478083554 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22497ff0: reopening flushed file at 1732478083557 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3900fc9b: reopening flushed file at 1732478083559 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=11, compaction requested=false at 1732478083562 (+3 ms)Writing region close event to WAL at 1732478083564 (+2 ms)Running coprocessor post-close hooks at 1732478083568 (+4 ms)Closed at 1732478083568 2024-11-24T19:54:43,568 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T19:54:43,684 INFO [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(976): stopping server 32a03c886245,33775,1732478080489; all regions closed. 2024-11-24T19:54:43,688 DEBUG [RS:0;32a03c886245:33775 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs 2024-11-24T19:54:43,688 INFO [RS:0;32a03c886245:33775 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32a03c886245%2C33775%2C1732478080489.meta:.meta(num 1732478081617) 2024-11-24T19:54:43,691 DEBUG [RS:0;32a03c886245:33775 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs 2024-11-24T19:54:43,691 INFO [RS:0;32a03c886245:33775 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32a03c886245%2C33775%2C1732478080489:(num 1732478081176) 2024-11-24T19:54:43,691 DEBUG [RS:0;32a03c886245:33775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:43,691 INFO [RS:0;32a03c886245:33775 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T19:54:43,692 INFO [RS:0;32a03c886245:33775 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T19:54:43,692 INFO [RS:0;32a03c886245:33775 {}] hbase.ChoreService(370): Chore service for: regionserver/32a03c886245:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T19:54:43,692 INFO [RS:0;32a03c886245:33775 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T19:54:43,692 INFO [regionserver/32a03c886245:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T19:54:43,693 INFO [RS:0;32a03c886245:33775 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33775 2024-11-24T19:54:43,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T19:54:43,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32a03c886245,33775,1732478080489 2024-11-24T19:54:43,715 INFO [RS:0;32a03c886245:33775 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T19:54:43,723 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32a03c886245,33775,1732478080489] 2024-11-24T19:54:43,731 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32a03c886245,33775,1732478080489 already deleted, retry=false 2024-11-24T19:54:43,731 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 32a03c886245,33775,1732478080489 on 32a03c886245,41101,1732478080355 2024-11-24T19:54:43,737 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 32a03c886245,33775,1732478080489, splitWal=true, meta=true 2024-11-24T19:54:43,740 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 32a03c886245,33775,1732478080489, splitWal=true, meta=true 2024-11-24T19:54:43,740 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 32a03c886245,33775,1732478080489 (carryingMeta=true) 32a03c886245,33775,1732478080489/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3738dcdf[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T19:54:43,741 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 32a03c886245,33775,1732478080489, splitWal=true, meta=true, isMeta: true 2024-11-24T19:54:43,742 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,33775,1732478080489-splitting 2024-11-24T19:54:43,742 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,33775,1732478080489-splitting dir is empty, no logs to split. 2024-11-24T19:54:43,742 INFO [PEWorker-2 {}] master.SplitWALManager(105): 32a03c886245,33775,1732478080489 WAL count=0, meta=true 2024-11-24T19:54:43,745 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,33775,1732478080489-splitting dir is empty, no logs to split. 2024-11-24T19:54:43,745 INFO [PEWorker-2 {}] master.SplitWALManager(105): 32a03c886245,33775,1732478080489 WAL count=0, meta=true 2024-11-24T19:54:43,745 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 32a03c886245,33775,1732478080489 WAL splitting is done? wals=0, meta=true 2024-11-24T19:54:43,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T19:54:43,749 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T19:54:43,750 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T19:54:43,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:43,823 INFO [RS:0;32a03c886245:33775 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T19:54:43,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33775-0x1016fb328190001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:43,823 INFO [RS:0;32a03c886245:33775 {}] regionserver.HRegionServer(1031): Exiting; stopping=32a03c886245,33775,1732478080489; zookeeper connection closed. 2024-11-24T19:54:43,824 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@311e944c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@311e944c 2024-11-24T19:54:43,900 DEBUG [32a03c886245:41101 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T19:54:43,901 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=32a03c886245,38051,1732478080521 2024-11-24T19:54:43,902 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32a03c886245,38051,1732478080521, state=OPENING 2024-11-24T19:54:43,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:54:43,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:54:43,923 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T19:54:43,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:43,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:43,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=32a03c886245,38051,1732478080521}] 2024-11-24T19:54:43,982 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32a03c886245:0 server-side Connection retries=45 2024-11-24T19:54:43,982 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:43,983 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:43,983 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T19:54:43,983 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:54:43,983 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T19:54:43,983 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T19:54:43,983 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T19:54:43,983 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45463 2024-11-24T19:54:43,985 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45463 connecting to ZooKeeper ensemble=127.0.0.1:54759 2024-11-24T19:54:43,985 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:43,985 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:44,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454630x0, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T19:54:44,015 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-24T19:54:44,015 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45463-0x1016fb328190003, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-24T19:54:44,015 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45463-0x1016fb328190003 connected 2024-11-24T19:54:44,016 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T19:54:44,017 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.cache 2024-11-24T19:54:44,017 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T19:54:44,017 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T19:54:44,017 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T19:54:44,017 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T19:54:44,017 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T19:54:44,018 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T19:54:44,018 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45463-0x1016fb328190003, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T19:54:44,019 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-24T19:54:44,019 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-24T19:54:44,020 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45463-0x1016fb328190003, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T19:54:44,022 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45463 2024-11-24T19:54:44,023 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-24T19:54:44,023 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T19:54:44,024 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-24T19:54:44,024 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.persistence is complete. 2024-11-24T19:54:44,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45463 2024-11-24T19:54:44,027 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45463 2024-11-24T19:54:44,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45463 2024-11-24T19:54:44,031 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45463 2024-11-24T19:54:44,032 INFO [RS:2;32a03c886245:45463 {}] regionserver.HRegionServer(746): ClusterId : c5ef859f-2a7e-4fa3-96ae-e7c0023a9b60 2024-11-24T19:54:44,032 DEBUG [RS:2;32a03c886245:45463 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T19:54:44,041 DEBUG [RS:2;32a03c886245:45463 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T19:54:44,041 DEBUG [RS:2;32a03c886245:45463 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T19:54:44,049 DEBUG [RS:2;32a03c886245:45463 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T19:54:44,049 DEBUG [RS:2;32a03c886245:45463 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4510b3e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32a03c886245/172.17.0.2:0 2024-11-24T19:54:44,060 DEBUG [RS:2;32a03c886245:45463 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;32a03c886245:45463 2024-11-24T19:54:44,060 INFO [RS:2;32a03c886245:45463 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T19:54:44,060 INFO [RS:2;32a03c886245:45463 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T19:54:44,060 DEBUG [RS:2;32a03c886245:45463 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T19:54:44,061 INFO [RS:2;32a03c886245:45463 {}] regionserver.HRegionServer(2659): reportForDuty to master=32a03c886245,41101,1732478080355 with port=45463, startcode=1732478083982 2024-11-24T19:54:44,061 DEBUG [RS:2;32a03c886245:45463 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T19:54:44,064 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45317, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T19:54:44,064 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41101 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32a03c886245,45463,1732478083982 2024-11-24T19:54:44,064 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41101 {}] master.ServerManager(517): Registering regionserver=32a03c886245,45463,1732478083982 2024-11-24T19:54:44,066 DEBUG [RS:2;32a03c886245:45463 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be 2024-11-24T19:54:44,066 DEBUG [RS:2;32a03c886245:45463 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T19:54:44,066 DEBUG [RS:2;32a03c886245:45463 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T19:54:44,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T19:54:44,073 DEBUG [RS:2;32a03c886245:45463 {}] zookeeper.ZKUtil(111): regionserver:45463-0x1016fb328190003, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32a03c886245,45463,1732478083982 2024-11-24T19:54:44,073 WARN [RS:2;32a03c886245:45463 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T19:54:44,074 INFO [RS:2;32a03c886245:45463 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:54:44,074 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32a03c886245,45463,1732478083982] 2024-11-24T19:54:44,074 DEBUG [RS:2;32a03c886245:45463 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,45463,1732478083982 2024-11-24T19:54:44,077 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T19:54:44,078 INFO [RS:2;32a03c886245:45463 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T19:54:44,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49731, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T19:54:44,081 INFO [RS:2;32a03c886245:45463 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T19:54:44,081 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T19:54:44,081 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:54:44,081 INFO [RS:2;32a03c886245:45463 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T19:54:44,081 INFO [RS:2;32a03c886245:45463 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:44,081 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T19:54:44,082 INFO [RS:2;32a03c886245:45463 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T19:54:44,082 INFO [RS:2;32a03c886245:45463 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T19:54:44,082 INFO [RS:2;32a03c886245:45463 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:44,082 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32a03c886245%2C38051%2C1732478080521.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,38051,1732478080521, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs, maxLogs=55 2024-11-24T19:54:44,082 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:44,082 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:44,082 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:44,083 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:44,083 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:44,083 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32a03c886245:0, corePoolSize=2, maxPoolSize=2 2024-11-24T19:54:44,083 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:44,083 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:44,083 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:44,083 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:44,083 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:44,083 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:54:44,083 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32a03c886245:0, corePoolSize=3, maxPoolSize=3 2024-11-24T19:54:44,083 DEBUG [RS:2;32a03c886245:45463 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32a03c886245:0, corePoolSize=3, maxPoolSize=3 2024-11-24T19:54:44,086 INFO [RS:2;32a03c886245:45463 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:44,086 INFO [RS:2;32a03c886245:45463 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:44,086 INFO [RS:2;32a03c886245:45463 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:44,086 INFO [RS:2;32a03c886245:45463 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:44,086 INFO [RS:2;32a03c886245:45463 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:44,086 INFO [RS:2;32a03c886245:45463 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,45463,1732478083982-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T19:54:44,099 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,38051,1732478080521/32a03c886245%2C38051%2C1732478080521.meta.1732478084083.meta 2024-11-24T19:54:44,099 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T19:54:44,099 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:54:44,099 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T19:54:44,099 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T19:54:44,100 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T19:54:44,100 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T19:54:44,100 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:44,100 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T19:54:44,100 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T19:54:44,101 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T19:54:44,101 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T19:54:44,101 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:44,102 INFO [RS:2;32a03c886245:45463 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T19:54:44,102 INFO [RS:2;32a03c886245:45463 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,45463,1732478083982-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:44,102 INFO [RS:2;32a03c886245:45463 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:44,102 INFO [RS:2;32a03c886245:45463 {}] regionserver.Replication(171): 32a03c886245,45463,1732478083982 started 2024-11-24T19:54:44,103 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/ef5de49693234ef2a6f22b64958f54d7, delay=978 ms 2024-11-24T19:54:44,104 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/ef5de49693234ef2a6f22b64958f54d7 2024-11-24T19:54:44,104 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:44,104 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T19:54:44,105 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T19:54:44,105 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:44,108 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/ns/5d425863b25a41a9b1a7e80eca05a7c8, delay=932 ms 2024-11-24T19:54:44,109 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/ns/5d425863b25a41a9b1a7e80eca05a7c8 2024-11-24T19:54:44,109 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:44,109 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T19:54:44,110 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T19:54:44,110 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:44,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:44,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T19:54:44,111 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T19:54:44,111 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:44,113 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/table/fd954b578a2f44489c6c891a97ac767f, delay=909 ms 2024-11-24T19:54:44,114 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/table/fd954b578a2f44489c6c891a97ac767f 2024-11-24T19:54:44,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:54:44,114 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T19:54:44,115 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740 2024-11-24T19:54:44,115 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740 2024-11-24T19:54:44,115 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T19:54:44,115 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T19:54:44,116 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T19:54:44,117 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T19:54:44,117 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74508075, jitterRate=0.11025683581829071}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T19:54:44,117 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T19:54:44,118 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732478084100Writing region info on filesystem at 1732478084100Initializing all the Stores at 1732478084100Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478084100Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478084101 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478084101Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478084101Cleaning up temporary data from old regions at 1732478084115 (+14 ms)Running coprocessor post-open hooks at 1732478084117 (+2 ms)Region opened successfully at 1732478084118 (+1 ms) 2024-11-24T19:54:44,119 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732478084077 2024-11-24T19:54:44,120 INFO [RS:2;32a03c886245:45463 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:54:44,120 INFO [RS:2;32a03c886245:45463 {}] regionserver.HRegionServer(1482): Serving as 32a03c886245,45463,1732478083982, RpcServer on 32a03c886245/172.17.0.2:45463, sessionid=0x1016fb328190003 2024-11-24T19:54:44,120 DEBUG [RS:2;32a03c886245:45463 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T19:54:44,120 DEBUG [RS:2;32a03c886245:45463 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32a03c886245,45463,1732478083982 2024-11-24T19:54:44,120 DEBUG [RS:2;32a03c886245:45463 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a03c886245,45463,1732478083982' 2024-11-24T19:54:44,120 DEBUG [RS:2;32a03c886245:45463 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T19:54:44,121 DEBUG [RS:2;32a03c886245:45463 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T19:54:44,121 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T19:54:44,121 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T19:54:44,122 DEBUG [RS:2;32a03c886245:45463 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T19:54:44,122 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=32a03c886245,38051,1732478080521 2024-11-24T19:54:44,122 DEBUG [RS:2;32a03c886245:45463 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T19:54:44,122 DEBUG [RS:2;32a03c886245:45463 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32a03c886245,45463,1732478083982 2024-11-24T19:54:44,122 DEBUG [RS:2;32a03c886245:45463 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a03c886245,45463,1732478083982' 2024-11-24T19:54:44,122 DEBUG [RS:2;32a03c886245:45463 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T19:54:44,122 DEBUG [RS:2;32a03c886245:45463 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T19:54:44,123 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32a03c886245,38051,1732478080521, state=OPEN 2024-11-24T19:54:44,123 DEBUG [RS:2;32a03c886245:45463 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T19:54:44,123 INFO [RS:2;32a03c886245:45463 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T19:54:44,123 INFO [RS:2;32a03c886245:45463 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T19:54:44,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45463-0x1016fb328190003, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:54:44,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:54:44,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:54:44,148 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=32a03c886245,38051,1732478080521 2024-11-24T19:54:44,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:44,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:44,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:54:44,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-24T19:54:44,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=32a03c886245,38051,1732478080521 in 225 msec 2024-11-24T19:54:44,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T19:54:44,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 403 msec 2024-11-24T19:54:44,152 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 32a03c886245,33775,1732478080489 had 2 regions 2024-11-24T19:54:44,153 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 32a03c886245,33775,1732478080489, splitWal=true, meta=true, isMeta: false 2024-11-24T19:54:44,154 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,33775,1732478080489-splitting dir is empty, no logs to split. 2024-11-24T19:54:44,154 INFO [PEWorker-2 {}] master.SplitWALManager(105): 32a03c886245,33775,1732478080489 WAL count=0, meta=false 2024-11-24T19:54:44,154 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,33775,1732478080489-splitting dir is empty, no logs to split. 2024-11-24T19:54:44,154 INFO [PEWorker-2 {}] master.SplitWALManager(105): 32a03c886245,33775,1732478080489 WAL count=0, meta=false 2024-11-24T19:54:44,155 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 32a03c886245,33775,1732478080489 WAL splitting is done? wals=0, meta=false 2024-11-24T19:54:44,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732478081761, region=f43868f9acd93b0e48aedd321a42eb23, ASSIGN}] 2024-11-24T19:54:44,157 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732478081761, region=f43868f9acd93b0e48aedd321a42eb23, ASSIGN 2024-11-24T19:54:44,158 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732478081761, region=f43868f9acd93b0e48aedd321a42eb23, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T19:54:44,224 INFO [RS:2;32a03c886245:45463 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T19:54:44,224 INFO [RS:2;32a03c886245:45463 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32a03c886245%2C45463%2C1732478083982, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,45463,1732478083982, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs, maxLogs=55 2024-11-24T19:54:44,244 INFO [RS:2;32a03c886245:45463 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/WALs/32a03c886245,45463,1732478083982/32a03c886245%2C45463%2C1732478083982.1732478084225 2024-11-24T19:54:44,244 DEBUG [RS:2;32a03c886245:45463 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T19:54:44,308 DEBUG [32a03c886245:41101 {}] balancer.BalancerClusterState(204): Hosts are {32a03c886245=0} racks are {/default-rack=0} 2024-11-24T19:54:44,309 DEBUG [32a03c886245:41101 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T19:54:44,309 DEBUG [32a03c886245:41101 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T19:54:44,309 DEBUG [32a03c886245:41101 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T19:54:44,309 DEBUG [32a03c886245:41101 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T19:54:44,309 INFO [32a03c886245:41101 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T19:54:44,309 INFO [32a03c886245:41101 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T19:54:44,309 DEBUG [32a03c886245:41101 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T19:54:44,309 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f43868f9acd93b0e48aedd321a42eb23, regionState=OPENING, regionLocation=32a03c886245,38051,1732478080521 2024-11-24T19:54:44,311 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 32a03c886245:33775 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 32a03c886245/172.17.0.2:33775 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T19:54:44,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=32a03c886245,33775,1732478080489, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=32a03c886245,33775,1732478080489, seqNum=-1, error=java.net.ConnectException: Call to address=32a03c886245:33775 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 32a03c886245/172.17.0.2:33775 2024-11-24T19:54:44,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=32a03c886245,33775,1732478080489, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-24T19:54:44,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=32a03c886245,33775,1732478080489, seqNum=-1 from cache 2024-11-24T19:54:44,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 32a03c886245:33775 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 32a03c886245/172.17.0.2:33775 2024-11-24T19:54:44,419 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T19:54:44,419 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a03c886245,38051,1732478080521, seqNum=-1] 2024-11-24T19:54:44,420 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T19:54:44,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34825, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T19:54:44,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732478081761, region=f43868f9acd93b0e48aedd321a42eb23, ASSIGN because future has completed 2024-11-24T19:54:44,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f43868f9acd93b0e48aedd321a42eb23, server=32a03c886245,38051,1732478080521}] 2024-11-24T19:54:44,524 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T19:54:44,524 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T19:54:44,526 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T19:54:44,526 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T19:54:44,526 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T19:54:44,526 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T19:54:44,584 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:44,585 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => f43868f9acd93b0e48aedd321a42eb23, NAME => 'testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:54:44,586 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732478081761 f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,586 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:54:44,587 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,587 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,588 INFO [StoreOpener-f43868f9acd93b0e48aedd321a42eb23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,590 INFO [StoreOpener-f43868f9acd93b0e48aedd321a42eb23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f43868f9acd93b0e48aedd321a42eb23 columnFamilyName family 2024-11-24T19:54:44,590 DEBUG [StoreOpener-f43868f9acd93b0e48aedd321a42eb23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:54:44,592 DEBUG [StoreFileOpener-f43868f9acd93b0e48aedd321a42eb23-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/family/8523cab502bc413aa2c43a35f5daef84, delay=950 ms 2024-11-24T19:54:44,593 DEBUG [StoreOpener-f43868f9acd93b0e48aedd321a42eb23-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/family/8523cab502bc413aa2c43a35f5daef84 2024-11-24T19:54:44,593 INFO [StoreOpener-f43868f9acd93b0e48aedd321a42eb23-1 {}] regionserver.HStore(327): Store=f43868f9acd93b0e48aedd321a42eb23/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:54:44,593 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,594 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,594 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,594 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,594 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,595 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,596 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened f43868f9acd93b0e48aedd321a42eb23; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64299065, jitterRate=-0.04186926782131195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:54:44,596 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,597 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for f43868f9acd93b0e48aedd321a42eb23: Running coprocessor pre-open hook at 1732478084587Writing region info on filesystem at 1732478084587Initializing all the Stores at 1732478084587Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478084588 (+1 ms)Cleaning up temporary data from old regions at 1732478084594 (+6 ms)Running coprocessor post-open hooks at 1732478084596 (+2 ms)Region opened successfully at 1732478084597 (+1 ms) 2024-11-24T19:54:44,597 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23., pid=11, masterSystemTime=1732478084581 2024-11-24T19:54:44,600 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:44,600 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:44,600 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f43868f9acd93b0e48aedd321a42eb23, regionState=OPEN, openSeqNum=10, regionLocation=32a03c886245,38051,1732478080521 2024-11-24T19:54:44,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f43868f9acd93b0e48aedd321a42eb23, server=32a03c886245,38051,1732478080521 because future has completed 2024-11-24T19:54:44,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T19:54:44,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure f43868f9acd93b0e48aedd321a42eb23, server=32a03c886245,38051,1732478080521 in 175 msec 2024-11-24T19:54:44,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-24T19:54:44,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732478081761, region=f43868f9acd93b0e48aedd321a42eb23, ASSIGN in 449 msec 2024-11-24T19:54:44,607 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(291): removed crashed server 32a03c886245,33775,1732478080489 after splitting done 2024-11-24T19:54:44,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 32a03c886245,33775,1732478080489, splitWal=true, meta=true in 875 msec 2024-11-24T19:54:44,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T19:54:44,620 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T19:54:44,621 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:54:44,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:44,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:44,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T19:54:44,621 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T19:54:44,621 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1613277258, stopped=false 2024-11-24T19:54:44,621 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=32a03c886245,41101,1732478080355 2024-11-24T19:54:44,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T19:54:44,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45463-0x1016fb328190003, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T19:54:44,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T19:54:44,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:44,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45463-0x1016fb328190003, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:44,664 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T19:54:44,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:44,665 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T19:54:44,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:54:44,665 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:54:44,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:44,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:54:44,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45463-0x1016fb328190003, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:54:44,666 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32a03c886245,38051,1732478080521' ***** 2024-11-24T19:54:44,666 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T19:54:44,666 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32a03c886245,45463,1732478083982' ***** 2024-11-24T19:54:44,666 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T19:54:44,666 INFO [RS:1;32a03c886245:38051 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T19:54:44,666 INFO [RS:2;32a03c886245:45463 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T19:54:44,666 INFO [RS:1;32a03c886245:38051 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.persistence 2024-11-24T19:54:44,666 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T19:54:44,666 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T19:54:44,666 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T19:54:44,666 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T19:54:44,666 INFO [RS:2;32a03c886245:45463 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/627377cf-eb87-67d3-fc84-43a81b6039b3/bucket.persistence 2024-11-24T19:54:44,666 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T19:54:44,666 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T19:54:44,667 INFO [RS:1;32a03c886245:38051 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T19:54:44,667 INFO [RS:2;32a03c886245:45463 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T19:54:44,667 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T19:54:44,667 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T19:54:44,677 DEBUG [RS:2;32a03c886245:45463 {}] bucket.BucketCache(1487): Thread RS:2;32a03c886245:45463 started persisting bucket cache to file 2024-11-24T19:54:44,677 DEBUG [RS:1;32a03c886245:38051 {}] bucket.BucketCache(1487): Thread RS:1;32a03c886245:38051 started persisting bucket cache to file 2024-11-24T19:54:44,677 DEBUG [RS:2;32a03c886245:45463 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T19:54:44,677 DEBUG [RS:1;32a03c886245:38051 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T19:54:44,677 DEBUG [RS:2;32a03c886245:45463 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T19:54:44,677 DEBUG [RS:1;32a03c886245:38051 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T19:54:44,680 DEBUG [RS:1;32a03c886245:38051 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T19:54:44,680 DEBUG [RS:2;32a03c886245:45463 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T19:54:44,680 DEBUG [RS:1;32a03c886245:38051 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732478084677 2024-11-24T19:54:44,680 DEBUG [RS:2;32a03c886245:45463 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732478084677 2024-11-24T19:54:44,680 DEBUG [RS:1;32a03c886245:38051 {}] bucket.BucketCache(1509): Thread RS:1;32a03c886245:38051 finished persisting bucket cache to file, renaming 2024-11-24T19:54:44,680 DEBUG [RS:2;32a03c886245:45463 {}] bucket.BucketCache(1509): Thread RS:2;32a03c886245:45463 finished persisting bucket cache to file, renaming 2024-11-24T19:54:44,680 WARN [RS:2;32a03c886245:45463 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-24T19:54:44,680 INFO [RS:1;32a03c886245:38051 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T19:54:44,680 INFO [RS:2;32a03c886245:45463 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T19:54:44,681 INFO [RS:1;32a03c886245:38051 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T19:54:44,681 INFO [RS:2;32a03c886245:45463 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T19:54:44,681 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T19:54:44,681 INFO [RS:1;32a03c886245:38051 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T19:54:44,681 INFO [RS:2;32a03c886245:45463 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T19:54:44,681 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T19:54:44,681 INFO [RS:2;32a03c886245:45463 {}] regionserver.HRegionServer(959): stopping server 32a03c886245,45463,1732478083982 2024-11-24T19:54:44,681 INFO [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(3091): Received CLOSE for f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,681 INFO [RS:2;32a03c886245:45463 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T19:54:44,681 INFO [RS:2;32a03c886245:45463 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;32a03c886245:45463. 2024-11-24T19:54:44,681 INFO [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(959): stopping server 32a03c886245,38051,1732478080521 2024-11-24T19:54:44,681 DEBUG [RS:2;32a03c886245:45463 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:54:44,681 INFO [RS:1;32a03c886245:38051 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T19:54:44,681 DEBUG [RS:2;32a03c886245:45463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:44,681 INFO [RS:1;32a03c886245:38051 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;32a03c886245:38051. 2024-11-24T19:54:44,681 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T19:54:44,682 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f43868f9acd93b0e48aedd321a42eb23, disabling compactions & flushes 2024-11-24T19:54:44,682 INFO [RS:2;32a03c886245:45463 {}] regionserver.HRegionServer(976): stopping server 32a03c886245,45463,1732478083982; all regions closed. 2024-11-24T19:54:44,682 DEBUG [RS:1;32a03c886245:38051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:54:44,682 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:44,682 DEBUG [RS:1;32a03c886245:38051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:44,682 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:44,682 INFO [RS:1;32a03c886245:38051 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T19:54:44,682 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. after waiting 0 ms 2024-11-24T19:54:44,682 INFO [RS:1;32a03c886245:38051 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T19:54:44,682 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:44,682 INFO [RS:1;32a03c886245:38051 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T19:54:44,682 INFO [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T19:54:44,683 INFO [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T19:54:44,683 DEBUG [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, f43868f9acd93b0e48aedd321a42eb23=testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23.} 2024-11-24T19:54:44,683 DEBUG [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f43868f9acd93b0e48aedd321a42eb23 2024-11-24T19:54:44,683 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T19:54:44,683 DEBUG [StoreFileCloser-f43868f9acd93b0e48aedd321a42eb23-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/family/8523cab502bc413aa2c43a35f5daef84 2024-11-24T19:54:44,683 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T19:54:44,683 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T19:54:44,683 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T19:54:44,683 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T19:54:44,684 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T19:54:44,684 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-24T19:54:44,685 DEBUG [RS:2;32a03c886245:45463 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs 2024-11-24T19:54:44,685 INFO [RS:2;32a03c886245:45463 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32a03c886245%2C45463%2C1732478083982:(num 1732478084225) 2024-11-24T19:54:44,685 DEBUG [RS:2;32a03c886245:45463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:44,685 INFO [RS:2;32a03c886245:45463 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T19:54:44,685 INFO [RS:2;32a03c886245:45463 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T19:54:44,685 INFO [RS:2;32a03c886245:45463 {}] hbase.ChoreService(370): Chore service for: regionserver/32a03c886245:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T19:54:44,686 INFO [RS:2;32a03c886245:45463 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T19:54:44,686 INFO [RS:2;32a03c886245:45463 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T19:54:44,686 INFO [RS:2;32a03c886245:45463 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T19:54:44,686 INFO [RS:2;32a03c886245:45463 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T19:54:44,686 INFO [RS:2;32a03c886245:45463 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45463 2024-11-24T19:54:44,687 INFO [regionserver/32a03c886245:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T19:54:44,687 INFO [regionserver/32a03c886245:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T19:54:44,688 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/default/testBlockEvictionOnGracefulStop1732478081761/f43868f9acd93b0e48aedd321a42eb23/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T19:54:44,689 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:44,689 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f43868f9acd93b0e48aedd321a42eb23: Waiting for close lock at 1732478084681Running coprocessor pre-close hooks at 1732478084682 (+1 ms)Disabling compacts and flushes for region at 1732478084682Disabling writes for close at 1732478084682Writing region close event to WAL at 1732478084684 (+2 ms)Running coprocessor post-close hooks at 1732478084689 (+5 ms)Closed at 1732478084689 2024-11-24T19:54:44,689 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23. 2024-11-24T19:54:44,693 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 59506c5bb06f4bf09063b9f0eb78304a 2024-11-24T19:54:44,706 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/info/59506c5bb06f4bf09063b9f0eb78304a is 209, key is testBlockEvictionOnGracefulStop1732478081761,,1732478081765.f43868f9acd93b0e48aedd321a42eb23./info:regioninfo/1732478084600/Put/seqid=0 2024-11-24T19:54:44,707 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/info/59506c5bb06f4bf09063b9f0eb78304a 2024-11-24T19:54:44,709 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/.tmp/info/59506c5bb06f4bf09063b9f0eb78304a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/59506c5bb06f4bf09063b9f0eb78304a 2024-11-24T19:54:44,710 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/59506c5bb06f4bf09063b9f0eb78304a, delay=970 ms 2024-11-24T19:54:44,711 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/59506c5bb06f4bf09063b9f0eb78304a, entries=8, sequenceid=19, filesize=6.6 K 2024-11-24T19:54:44,711 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 27ms, sequenceid=19, compaction requested=false 2024-11-24T19:54:44,712 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/ef5de49693234ef2a6f22b64958f54d7 2024-11-24T19:54:44,712 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/info/59506c5bb06f4bf09063b9f0eb78304a 2024-11-24T19:54:44,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T19:54:44,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45463-0x1016fb328190003, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32a03c886245,45463,1732478083982 2024-11-24T19:54:44,713 INFO [RS:2;32a03c886245:45463 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T19:54:44,713 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/ns/5d425863b25a41a9b1a7e80eca05a7c8 2024-11-24T19:54:44,713 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/table/fd954b578a2f44489c6c891a97ac767f 2024-11-24T19:54:44,713 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32a03c886245,45463,1732478083982] 2024-11-24T19:54:44,718 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-24T19:54:44,718 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T19:54:44,718 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T19:54:44,719 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732478084683Running coprocessor pre-close hooks at 1732478084683Disabling compacts and flushes for region at 1732478084683Disabling writes for close at 1732478084683Obtaining lock to block concurrent updates at 1732478084684 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732478084684Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732478084685 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732478084685Flushing 1588230740/info: creating writer at 1732478084685Flushing 1588230740/info: appending metadata at 1732478084706 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732478084706Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a474fa4: reopening flushed file at 1732478084709 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 27ms, sequenceid=19, compaction requested=false at 1732478084711 (+2 ms)Writing region close event to WAL at 1732478084714 (+3 ms)Running coprocessor post-close hooks at 1732478084718 (+4 ms)Closed at 1732478084718 2024-11-24T19:54:44,719 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T19:54:44,731 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32a03c886245,45463,1732478083982 already deleted, retry=false 2024-11-24T19:54:44,731 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 32a03c886245,45463,1732478083982 expired; onlineServers=1 2024-11-24T19:54:44,740 INFO [regionserver/32a03c886245:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T19:54:44,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45463-0x1016fb328190003, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:44,823 INFO [RS:2;32a03c886245:45463 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T19:54:44,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45463-0x1016fb328190003, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:44,823 INFO [RS:2;32a03c886245:45463 {}] regionserver.HRegionServer(1031): Exiting; stopping=32a03c886245,45463,1732478083982; zookeeper connection closed. 2024-11-24T19:54:44,824 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@50256873 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@50256873 2024-11-24T19:54:44,883 INFO [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(976): stopping server 32a03c886245,38051,1732478080521; all regions closed. 2024-11-24T19:54:44,888 DEBUG [RS:1;32a03c886245:38051 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs 2024-11-24T19:54:44,889 INFO [RS:1;32a03c886245:38051 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32a03c886245%2C38051%2C1732478080521.meta:.meta(num 1732478084083) 2024-11-24T19:54:44,891 DEBUG [RS:1;32a03c886245:38051 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/oldWALs 2024-11-24T19:54:44,892 INFO [RS:1;32a03c886245:38051 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32a03c886245%2C38051%2C1732478080521:(num 1732478081176) 2024-11-24T19:54:44,892 DEBUG [RS:1;32a03c886245:38051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:54:44,892 INFO [RS:1;32a03c886245:38051 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T19:54:44,892 INFO [RS:1;32a03c886245:38051 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T19:54:44,892 INFO [RS:1;32a03c886245:38051 {}] hbase.ChoreService(370): Chore service for: regionserver/32a03c886245:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T19:54:44,892 INFO [RS:1;32a03c886245:38051 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T19:54:44,893 INFO [regionserver/32a03c886245:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T19:54:44,893 INFO [RS:1;32a03c886245:38051 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38051 2024-11-24T19:54:44,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T19:54:44,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32a03c886245,38051,1732478080521 2024-11-24T19:54:44,923 INFO [RS:1;32a03c886245:38051 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T19:54:44,931 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32a03c886245,38051,1732478080521] 2024-11-24T19:54:44,939 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32a03c886245,38051,1732478080521 already deleted, retry=false 2024-11-24T19:54:44,939 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 32a03c886245,38051,1732478080521 expired; onlineServers=0 2024-11-24T19:54:44,939 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '32a03c886245,41101,1732478080355' ***** 2024-11-24T19:54:44,939 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T19:54:44,939 INFO [M:0;32a03c886245:41101 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T19:54:44,939 INFO [M:0;32a03c886245:41101 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T19:54:44,940 DEBUG [M:0;32a03c886245:41101 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T19:54:44,940 DEBUG [M:0;32a03c886245:41101 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T19:54:44,940 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T19:54:44,940 DEBUG [master/32a03c886245:0:becomeActiveMaster-HFileCleaner.large.0-1732478080943 {}] cleaner.HFileCleaner(306): Exit Thread[master/32a03c886245:0:becomeActiveMaster-HFileCleaner.large.0-1732478080943,5,FailOnTimeoutGroup] 2024-11-24T19:54:44,940 DEBUG [master/32a03c886245:0:becomeActiveMaster-HFileCleaner.small.0-1732478080943 {}] cleaner.HFileCleaner(306): Exit Thread[master/32a03c886245:0:becomeActiveMaster-HFileCleaner.small.0-1732478080943,5,FailOnTimeoutGroup] 2024-11-24T19:54:44,940 INFO [M:0;32a03c886245:41101 {}] hbase.ChoreService(370): Chore service for: master/32a03c886245:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T19:54:44,940 INFO [M:0;32a03c886245:41101 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T19:54:44,940 DEBUG [M:0;32a03c886245:41101 {}] master.HMaster(1795): Stopping service threads 2024-11-24T19:54:44,940 INFO [M:0;32a03c886245:41101 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T19:54:44,940 INFO [M:0;32a03c886245:41101 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T19:54:44,941 INFO [M:0;32a03c886245:41101 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T19:54:44,941 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T19:54:44,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T19:54:44,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:54:44,948 DEBUG [M:0;32a03c886245:41101 {}] zookeeper.ZKUtil(347): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T19:54:44,948 WARN [M:0;32a03c886245:41101 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T19:54:44,948 INFO [M:0;32a03c886245:41101 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/.lastflushedseqids 2024-11-24T19:54:44,956 INFO [M:0;32a03c886245:41101 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T19:54:44,957 INFO [M:0;32a03c886245:41101 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T19:54:44,957 DEBUG [M:0;32a03c886245:41101 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T19:54:44,957 INFO [M:0;32a03c886245:41101 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:44,957 DEBUG [M:0;32a03c886245:41101 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:44,957 DEBUG [M:0;32a03c886245:41101 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T19:54:44,957 DEBUG [M:0;32a03c886245:41101 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:44,957 INFO [M:0;32a03c886245:41101 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-24T19:54:44,966 DEBUG [M:0;32a03c886245:41101 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a8c54c47d07c4844bc53364050abf502 2024-11-24T19:54:44,978 DEBUG [M:0;32a03c886245:41101 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a8c54c47d07c4844bc53364050abf502 is 82, key is hbase:meta,,1/info:regioninfo/1732478084122/Put/seqid=0 2024-11-24T19:54:44,979 INFO [M:0;32a03c886245:41101 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a8c54c47d07c4844bc53364050abf502 2024-11-24T19:54:44,984 DEBUG [M:0;32a03c886245:41101 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 77a5d1244c9b4860a9edeb28f90e3127 2024-11-24T19:54:44,997 DEBUG [M:0;32a03c886245:41101 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/77a5d1244c9b4860a9edeb28f90e3127 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732478082144/Put/seqid=0 2024-11-24T19:54:44,998 INFO [M:0;32a03c886245:41101 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/77a5d1244c9b4860a9edeb28f90e3127 2024-11-24T19:54:45,004 DEBUG [M:0;32a03c886245:41101 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 83f311095a404f7f8f9185af5115d44a 2024-11-24T19:54:45,018 DEBUG [M:0;32a03c886245:41101 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/83f311095a404f7f8f9185af5115d44a is 69, key is 32a03c886245,38051,1732478080521/rs:state/1732478080977/Put/seqid=0 2024-11-24T19:54:45,018 INFO [M:0;32a03c886245:41101 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/83f311095a404f7f8f9185af5115d44a 2024-11-24T19:54:45,020 INFO [M:0;32a03c886245:41101 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 83f311095a404f7f8f9185af5115d44a 2024-11-24T19:54:45,020 DEBUG [M:0;32a03c886245:41101 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a8c54c47d07c4844bc53364050abf502 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a8c54c47d07c4844bc53364050abf502 2024-11-24T19:54:45,022 INFO [M:0;32a03c886245:41101 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a8c54c47d07c4844bc53364050abf502, entries=14, sequenceid=102, filesize=5.9 K 2024-11-24T19:54:45,022 DEBUG [M:0;32a03c886245:41101 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/77a5d1244c9b4860a9edeb28f90e3127 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/77a5d1244c9b4860a9edeb28f90e3127 2024-11-24T19:54:45,024 INFO [M:0;32a03c886245:41101 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/77a5d1244c9b4860a9edeb28f90e3127, entries=11, sequenceid=102, filesize=6.5 K 2024-11-24T19:54:45,024 DEBUG [M:0;32a03c886245:41101 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/83f311095a404f7f8f9185af5115d44a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/83f311095a404f7f8f9185af5115d44a 2024-11-24T19:54:45,025 INFO [M:0;32a03c886245:41101 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 83f311095a404f7f8f9185af5115d44a 2024-11-24T19:54:45,025 INFO [M:0;32a03c886245:41101 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c139ee3-a67d-1330-c7d0-b45711ecb45b/7b83fd75-9cad-5ee8-fb8b-66d2d6a3a2be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/83f311095a404f7f8f9185af5115d44a, entries=3, sequenceid=102, filesize=5.3 K 2024-11-24T19:54:45,026 INFO [M:0;32a03c886245:41101 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 69ms, sequenceid=102, compaction requested=false 2024-11-24T19:54:45,028 INFO [M:0;32a03c886245:41101 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:54:45,028 DEBUG [M:0;32a03c886245:41101 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732478084957Disabling compacts and flushes for region at 1732478084957Disabling writes for close at 1732478084957Obtaining lock to block concurrent updates at 1732478084957Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732478084957Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1732478084958 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732478084958Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732478084958Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732478084978 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732478084978Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732478084980 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732478084997 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732478084997Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732478084999 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732478085017 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732478085017Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55cf124d: reopening flushed file at 1732478085020 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c7e14a9: reopening flushed file at 1732478085022 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59445c20: reopening flushed file at 1732478085024 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 69ms, sequenceid=102, compaction requested=false at 1732478085026 (+2 ms)Writing region close event to WAL at 1732478085028 (+2 ms)Closed at 1732478085028 2024-11-24T19:54:45,029 INFO [M:0;32a03c886245:41101 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T19:54:45,029 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T19:54:45,029 INFO [M:0;32a03c886245:41101 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41101 2024-11-24T19:54:45,029 INFO [M:0;32a03c886245:41101 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T19:54:45,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:45,031 INFO [RS:1;32a03c886245:38051 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T19:54:45,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38051-0x1016fb328190002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:45,031 INFO [RS:1;32a03c886245:38051 {}] regionserver.HRegionServer(1031): Exiting; stopping=32a03c886245,38051,1732478080521; zookeeper connection closed. 2024-11-24T19:54:45,032 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2e8db77f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2e8db77f 2024-11-24T19:54:45,032 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T19:54:45,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:45,148 INFO [M:0;32a03c886245:41101 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T19:54:45,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41101-0x1016fb328190000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:54:45,150 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T19:54:45,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T19:54:45,151 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:54:45,157 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=530 (was 576), ProcessCount=11 (was 11), AvailableMemoryMB=11100 (was 11228)