2024-11-13 13:48:01,843 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-13 13:48:01,869 main DEBUG Took 0.023242 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 13:48:01,870 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 13:48:01,870 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 13:48:01,871 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 13:48:01,872 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,881 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 13:48:01,894 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,896 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,897 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,898 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,898 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,899 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,900 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,900 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,901 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,901 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,903 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,903 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,904 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,904 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,904 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,905 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,905 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,906 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,906 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,906 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,907 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,907 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,907 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 13:48:01,908 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,908 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 13:48:01,910 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 13:48:01,912 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 13:48:01,915 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 13:48:01,915 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 13:48:01,917 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 13:48:01,917 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 13:48:01,929 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 13:48:01,933 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 13:48:01,935 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 13:48:01,936 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 13:48:01,936 main DEBUG createAppenders(={Console}) 2024-11-13 13:48:01,937 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-13 13:48:01,938 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-13 13:48:01,938 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-13 13:48:01,939 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 13:48:01,939 main DEBUG OutputStream closed 2024-11-13 13:48:01,940 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 13:48:01,940 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 13:48:01,940 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-13 13:48:02,028 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 13:48:02,031 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 13:48:02,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 13:48:02,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 13:48:02,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 13:48:02,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 13:48:02,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 13:48:02,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 13:48:02,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 13:48:02,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 13:48:02,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 13:48:02,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 13:48:02,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 13:48:02,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 13:48:02,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 13:48:02,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 13:48:02,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 13:48:02,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 13:48:02,041 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 13:48:02,042 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-13 13:48:02,042 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 13:48:02,043 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-13T13:48:02,056 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-13 13:48:02,059 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 13:48:02,059 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T13:48:02,326 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4 2024-11-13T13:48:02,351 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=616, ProcessCount=11, AvailableMemoryMB=3131 2024-11-13T13:48:02,458 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T13:48:02,565 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:02,581 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/cluster_3734be54-941a-0122-0121-5b18bc377cef, deleteOnExit=true 2024-11-13T13:48:02,677 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/cluster_3734be54-941a-0122-0121-5b18bc377cef/zookeeper_0, clientPort=57932, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/cluster_3734be54-941a-0122-0121-5b18bc377cef/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/cluster_3734be54-941a-0122-0121-5b18bc377cef/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T13:48:02,690 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57932 2024-11-13T13:48:02,690 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:02,812 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2 with version=8 2024-11-13T13:48:02,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/hbase-staging 2024-11-13T13:48:02,917 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T13:48:03,208 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0b62828576b1:0 server-side Connection retries=45 2024-11-13T13:48:03,220 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:03,222 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:03,233 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T13:48:03,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:03,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T13:48:03,412 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T13:48:03,512 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T13:48:03,552 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T13:48:03,563 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T13:48:03,567 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T13:48:03,614 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 19928 (auto-detected) 2024-11-13T13:48:03,615 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T13:48:03,636 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40603 2024-11-13T13:48:03,661 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40603 connecting to ZooKeeper ensemble=127.0.0.1:57932 2024-11-13T13:48:03,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:406030x0, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T13:48:03,701 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40603-0x1003966037f0000 connected 2024-11-13T13:48:03,731 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:03,732 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:03,741 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T13:48:03,745 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2, hbase.cluster.distributed=false 2024-11-13T13:48:03,774 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T13:48:03,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40603 2024-11-13T13:48:03,784 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40603 2024-11-13T13:48:03,784 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40603 2024-11-13T13:48:03,785 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40603 2024-11-13T13:48:03,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40603 2024-11-13T13:48:03,887 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b62828576b1:0 server-side Connection retries=45 2024-11-13T13:48:03,889 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:03,889 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:03,889 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T13:48:03,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:03,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T13:48:03,893 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T13:48:03,895 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T13:48:03,897 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40827 2024-11-13T13:48:03,899 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40827 connecting to ZooKeeper ensemble=127.0.0.1:57932 2024-11-13T13:48:03,900 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:03,901 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:03,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:408270x0, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T13:48:03,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40827-0x1003966037f0001 connected 2024-11-13T13:48:03,906 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T13:48:03,912 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T13:48:03,927 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/bucket.cache 2024-11-13T13:48:03,936 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T13:48:03,937 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T13:48:03,940 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T13:48:03,942 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T13:48:03,942 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-13T13:48:03,950 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T13:48:03,950 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T13:48:03,951 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/bucket.persistence is complete. 2024-11-13T13:48:03,958 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T13:48:03,961 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T13:48:03,967 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T13:48:03,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40827 2024-11-13T13:48:03,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40827 2024-11-13T13:48:03,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40827 2024-11-13T13:48:03,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40827 2024-11-13T13:48:03,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40827 2024-11-13T13:48:03,989 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b62828576b1:0 server-side Connection retries=45 2024-11-13T13:48:03,989 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:03,990 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:03,990 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T13:48:03,990 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:03,990 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T13:48:03,990 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T13:48:03,991 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T13:48:03,992 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36171 2024-11-13T13:48:03,994 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36171 connecting to ZooKeeper ensemble=127.0.0.1:57932 2024-11-13T13:48:03,995 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:03,995 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:03,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361710x0, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T13:48:04,000 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36171-0x1003966037f0002 connected 2024-11-13T13:48:04,000 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T13:48:04,001 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T13:48:04,004 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/bucket.cache 2024-11-13T13:48:04,004 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T13:48:04,005 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T13:48:04,005 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T13:48:04,005 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T13:48:04,006 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-13T13:48:04,006 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T13:48:04,006 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T13:48:04,006 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/bucket.persistence is complete. 2024-11-13T13:48:04,006 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T13:48:04,008 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T13:48:04,011 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T13:48:04,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36171 2024-11-13T13:48:04,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36171 2024-11-13T13:48:04,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36171 2024-11-13T13:48:04,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36171 2024-11-13T13:48:04,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36171 2024-11-13T13:48:04,045 DEBUG [M:0;0b62828576b1:40603 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0b62828576b1:40603 2024-11-13T13:48:04,046 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0b62828576b1,40603,1731505682995 2024-11-13T13:48:04,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T13:48:04,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T13:48:04,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T13:48:04,059 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0b62828576b1,40603,1731505682995 2024-11-13T13:48:04,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T13:48:04,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T13:48:04,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,093 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T13:48:04,094 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0b62828576b1,40603,1731505682995 from backup master directory 2024-11-13T13:48:04,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0b62828576b1,40603,1731505682995 2024-11-13T13:48:04,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T13:48:04,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T13:48:04,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T13:48:04,099 WARN [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T13:48:04,099 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0b62828576b1,40603,1731505682995 2024-11-13T13:48:04,102 INFO [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T13:48:04,104 INFO [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T13:48:04,114 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/hbase.id] with ID: 8b6f7d11-e80f-4ae3-a253-a3c3fe358ca7 2024-11-13T13:48:04,114 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/.tmp/hbase.id 2024-11-13T13:48:04,120 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/hbase.id] 2024-11-13T13:48:04,186 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:04,192 INFO [master/0b62828576b1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T13:48:04,211 INFO [master/0b62828576b1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-13T13:48:04,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,244 INFO [master/0b62828576b1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T13:48:04,247 INFO [master/0b62828576b1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T13:48:04,295 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T13:48:04,301 INFO [master/0b62828576b1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T13:48:04,362 INFO [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store 2024-11-13T13:48:04,384 INFO [master/0b62828576b1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T13:48:04,387 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:04,389 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T13:48:04,389 INFO [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:04,389 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:04,391 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T13:48:04,391 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:04,391 INFO [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:04,392 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731505684389Disabling compacts and flushes for region at 1731505684389Disabling writes for close at 1731505684391 (+2 ms)Writing region close event to WAL at 1731505684391Closed at 1731505684391 2024-11-13T13:48:04,393 WARN [master/0b62828576b1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/.initializing 2024-11-13T13:48:04,393 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/WALs/0b62828576b1,40603,1731505682995 2024-11-13T13:48:04,407 INFO [master/0b62828576b1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T13:48:04,420 INFO [master/0b62828576b1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b62828576b1%2C40603%2C1731505682995, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/WALs/0b62828576b1,40603,1731505682995, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/oldWALs, maxLogs=10 2024-11-13T13:48:04,460 INFO [master/0b62828576b1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/WALs/0b62828576b1,40603,1731505682995/0b62828576b1%2C40603%2C1731505682995.1731505684424 2024-11-13T13:48:04,461 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T13:48:04,461 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T13:48:04,462 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:04,465 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:04,466 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:04,507 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:04,540 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T13:48:04,545 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:04,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:04,548 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:04,554 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T13:48:04,555 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:04,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T13:48:04,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:04,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T13:48:04,560 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:04,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T13:48:04,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:04,566 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T13:48:04,566 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:04,567 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T13:48:04,567 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:04,571 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:04,571 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:04,575 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:04,575 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:04,579 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T13:48:04,581 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:04,591 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T13:48:04,592 INFO [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71845159, jitterRate=0.07057629525661469}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T13:48:04,599 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731505684479Initializing all the Stores at 1731505684479Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505684480 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505684481 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505684481Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505684481Cleaning up temporary data from old regions at 1731505684576 (+95 ms)Region opened successfully at 1731505684599 (+23 ms) 2024-11-13T13:48:04,601 INFO [master/0b62828576b1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T13:48:04,636 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604b4acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b62828576b1/172.17.0.3:0 2024-11-13T13:48:04,678 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T13:48:04,694 INFO [master/0b62828576b1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T13:48:04,695 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T13:48:04,699 INFO [master/0b62828576b1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T13:48:04,700 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T13:48:04,708 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-13T13:48:04,708 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T13:48:04,744 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T13:48:04,757 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T13:48:04,760 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T13:48:04,764 INFO [master/0b62828576b1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T13:48:04,766 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T13:48:04,769 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T13:48:04,772 INFO [master/0b62828576b1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T13:48:04,778 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T13:48:04,779 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T13:48:04,781 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T13:48:04,783 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T13:48:04,805 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T13:48:04,807 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T13:48:04,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T13:48:04,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T13:48:04,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T13:48:04,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,843 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0b62828576b1,40603,1731505682995, sessionid=0x1003966037f0000, setting cluster-up flag (Was=false) 2024-11-13T13:48:04,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,867 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T13:48:04,869 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b62828576b1,40603,1731505682995 2024-11-13T13:48:04,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:04,883 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T13:48:04,885 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b62828576b1,40603,1731505682995 2024-11-13T13:48:04,892 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T13:48:04,925 INFO [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(746): ClusterId : 8b6f7d11-e80f-4ae3-a253-a3c3fe358ca7 2024-11-13T13:48:04,925 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(746): ClusterId : 8b6f7d11-e80f-4ae3-a253-a3c3fe358ca7 2024-11-13T13:48:04,928 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T13:48:04,928 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T13:48:04,936 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T13:48:04,936 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T13:48:04,936 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T13:48:04,936 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T13:48:04,945 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T13:48:04,945 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T13:48:04,947 DEBUG [RS:0;0b62828576b1:40827 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4257fe9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b62828576b1/172.17.0.3:0 2024-11-13T13:48:04,947 DEBUG [RS:1;0b62828576b1:36171 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75937453, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b62828576b1/172.17.0.3:0 2024-11-13T13:48:04,967 DEBUG [RS:1;0b62828576b1:36171 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0b62828576b1:36171 2024-11-13T13:48:04,969 DEBUG [RS:0;0b62828576b1:40827 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0b62828576b1:40827 2024-11-13T13:48:04,971 INFO [RS:1;0b62828576b1:36171 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T13:48:04,971 INFO [RS:0;0b62828576b1:40827 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T13:48:04,971 INFO [RS:0;0b62828576b1:40827 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T13:48:04,971 INFO [RS:1;0b62828576b1:36171 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T13:48:04,971 DEBUG [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T13:48:04,971 DEBUG [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T13:48:04,975 INFO [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b62828576b1,40603,1731505682995 with port=40827, startcode=1731505683846 2024-11-13T13:48:04,975 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b62828576b1,40603,1731505682995 with port=36171, startcode=1731505683989 2024-11-13T13:48:04,991 DEBUG [RS:1;0b62828576b1:36171 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T13:48:04,991 DEBUG [RS:0;0b62828576b1:40827 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T13:48:04,991 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T13:48:05,004 INFO [master/0b62828576b1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T13:48:05,011 INFO [master/0b62828576b1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T13:48:05,017 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0b62828576b1,40603,1731505682995 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T13:48:05,026 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0b62828576b1:0, corePoolSize=5, maxPoolSize=5 2024-11-13T13:48:05,027 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0b62828576b1:0, corePoolSize=5, maxPoolSize=5 2024-11-13T13:48:05,027 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0b62828576b1:0, corePoolSize=5, maxPoolSize=5 2024-11-13T13:48:05,027 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0b62828576b1:0, corePoolSize=5, maxPoolSize=5 2024-11-13T13:48:05,028 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0b62828576b1:0, corePoolSize=10, maxPoolSize=10 2024-11-13T13:48:05,028 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,028 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0b62828576b1:0, corePoolSize=2, maxPoolSize=2 2024-11-13T13:48:05,029 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,036 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T13:48:05,041 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T13:48:05,046 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:05,046 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T13:48:05,077 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731505715076 2024-11-13T13:48:05,077 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35439, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T13:48:05,077 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36005, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T13:48:05,080 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T13:48:05,080 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T13:48:05,081 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2 2024-11-13T13:48:05,082 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T13:48:05,086 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T13:48:05,086 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T13:48:05,087 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T13:48:05,087 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T13:48:05,087 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40603 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T13:48:05,088 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,093 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T13:48:05,095 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T13:48:05,095 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T13:48:05,095 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40603 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T13:48:05,098 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T13:48:05,098 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T13:48:05,099 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:05,103 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T13:48:05,108 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0b62828576b1:0:becomeActiveMaster-HFileCleaner.large.0-1731505685100,5,FailOnTimeoutGroup] 2024-11-13T13:48:05,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T13:48:05,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:05,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:05,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T13:48:05,116 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0b62828576b1:0:becomeActiveMaster-HFileCleaner.small.0-1731505685109,5,FailOnTimeoutGroup] 2024-11-13T13:48:05,116 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,117 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T13:48:05,118 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,119 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,121 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T13:48:05,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:05,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:05,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T13:48:05,127 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T13:48:05,127 DEBUG [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T13:48:05,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:05,128 WARN [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T13:48:05,128 DEBUG [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T13:48:05,128 WARN [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T13:48:05,128 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:05,128 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T13:48:05,138 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T13:48:05,138 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:05,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:05,139 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T13:48:05,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740 2024-11-13T13:48:05,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740 2024-11-13T13:48:05,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T13:48:05,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T13:48:05,142 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T13:48:05,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T13:48:05,153 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T13:48:05,154 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65201388, jitterRate=-0.028423607349395752}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T13:48:05,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731505685099Initializing all the Stores at 1731505685100 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505685100Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505685103 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505685103Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505685103Cleaning up temporary data from old regions at 1731505685141 (+38 ms)Region opened successfully at 1731505685158 (+17 ms) 2024-11-13T13:48:05,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T13:48:05,159 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T13:48:05,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T13:48:05,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T13:48:05,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T13:48:05,161 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T13:48:05,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731505685159Disabling compacts and flushes for region at 1731505685159Disabling writes for close at 1731505685159Writing region close event to WAL at 1731505685161 (+2 ms)Closed at 1731505685161 2024-11-13T13:48:05,165 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T13:48:05,165 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T13:48:05,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T13:48:05,185 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T13:48:05,188 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T13:48:05,229 INFO [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b62828576b1,40603,1731505682995 with port=40827, startcode=1731505683846 2024-11-13T13:48:05,229 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b62828576b1,40603,1731505682995 with port=36171, startcode=1731505683989 2024-11-13T13:48:05,232 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40603 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b62828576b1,40827,1731505683846 2024-11-13T13:48:05,234 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40603 {}] master.ServerManager(517): Registering regionserver=0b62828576b1,40827,1731505683846 2024-11-13T13:48:05,242 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40603 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b62828576b1,36171,1731505683989 2024-11-13T13:48:05,243 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40603 {}] master.ServerManager(517): Registering regionserver=0b62828576b1,36171,1731505683989 2024-11-13T13:48:05,243 DEBUG [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2 2024-11-13T13:48:05,243 DEBUG [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T13:48:05,243 DEBUG [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T13:48:05,245 DEBUG [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2 2024-11-13T13:48:05,245 DEBUG [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T13:48:05,245 DEBUG [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T13:48:05,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T13:48:05,254 DEBUG [RS:1;0b62828576b1:36171 {}] zookeeper.ZKUtil(111): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b62828576b1,36171,1731505683989 2024-11-13T13:48:05,254 WARN [RS:1;0b62828576b1:36171 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T13:48:05,254 INFO [RS:1;0b62828576b1:36171 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T13:48:05,255 DEBUG [RS:0;0b62828576b1:40827 {}] zookeeper.ZKUtil(111): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b62828576b1,40827,1731505683846 2024-11-13T13:48:05,255 WARN [RS:0;0b62828576b1:40827 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T13:48:05,255 DEBUG [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,36171,1731505683989 2024-11-13T13:48:05,255 INFO [RS:0;0b62828576b1:40827 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T13:48:05,255 DEBUG [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,40827,1731505683846 2024-11-13T13:48:05,257 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b62828576b1,40827,1731505683846] 2024-11-13T13:48:05,257 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b62828576b1,36171,1731505683989] 2024-11-13T13:48:05,286 INFO [RS:1;0b62828576b1:36171 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T13:48:05,286 INFO [RS:0;0b62828576b1:40827 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T13:48:05,307 INFO [RS:0;0b62828576b1:40827 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T13:48:05,307 INFO [RS:1;0b62828576b1:36171 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T13:48:05,312 INFO [RS:0;0b62828576b1:40827 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T13:48:05,312 INFO [RS:1;0b62828576b1:36171 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T13:48:05,312 INFO [RS:1;0b62828576b1:36171 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,312 INFO [RS:0;0b62828576b1:40827 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,313 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T13:48:05,313 INFO [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T13:48:05,319 INFO [RS:1;0b62828576b1:36171 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T13:48:05,319 INFO [RS:0;0b62828576b1:40827 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T13:48:05,321 INFO [RS:1;0b62828576b1:36171 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,321 INFO [RS:0;0b62828576b1:40827 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,322 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,322 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,322 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,322 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,322 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,323 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,323 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,323 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b62828576b1:0, corePoolSize=2, maxPoolSize=2 2024-11-13T13:48:05,323 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,323 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,323 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,323 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,324 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,324 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,324 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b62828576b1:0, corePoolSize=2, maxPoolSize=2 2024-11-13T13:48:05,324 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,324 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,324 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,324 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,324 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,325 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,325 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b62828576b1:0, corePoolSize=3, maxPoolSize=3 2024-11-13T13:48:05,325 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,325 DEBUG [RS:1;0b62828576b1:36171 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b62828576b1:0, corePoolSize=3, maxPoolSize=3 2024-11-13T13:48:05,325 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,325 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:05,325 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b62828576b1:0, corePoolSize=3, maxPoolSize=3 2024-11-13T13:48:05,326 DEBUG [RS:0;0b62828576b1:40827 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b62828576b1:0, corePoolSize=3, maxPoolSize=3 2024-11-13T13:48:05,331 INFO [RS:1;0b62828576b1:36171 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,331 INFO [RS:1;0b62828576b1:36171 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,332 INFO [RS:1;0b62828576b1:36171 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,332 INFO [RS:1;0b62828576b1:36171 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,332 INFO [RS:1;0b62828576b1:36171 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,332 INFO [RS:1;0b62828576b1:36171 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,36171,1731505683989-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T13:48:05,332 INFO [RS:0;0b62828576b1:40827 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,333 INFO [RS:0;0b62828576b1:40827 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,333 INFO [RS:0;0b62828576b1:40827 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,333 INFO [RS:0;0b62828576b1:40827 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,333 INFO [RS:0;0b62828576b1:40827 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,333 INFO [RS:0;0b62828576b1:40827 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,40827,1731505683846-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T13:48:05,339 WARN [0b62828576b1:40603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T13:48:05,362 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T13:48:05,364 INFO [RS:1;0b62828576b1:36171 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,36171,1731505683989-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,365 INFO [RS:1;0b62828576b1:36171 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,365 INFO [RS:1;0b62828576b1:36171 {}] regionserver.Replication(171): 0b62828576b1,36171,1731505683989 started 2024-11-13T13:48:05,366 INFO [RS:0;0b62828576b1:40827 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T13:48:05,366 INFO [RS:0;0b62828576b1:40827 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,40827,1731505683846-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,366 INFO [RS:0;0b62828576b1:40827 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,366 INFO [RS:0;0b62828576b1:40827 {}] regionserver.Replication(171): 0b62828576b1,40827,1731505683846 started 2024-11-13T13:48:05,390 INFO [RS:0;0b62828576b1:40827 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,391 INFO [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(1482): Serving as 0b62828576b1,40827,1731505683846, RpcServer on 0b62828576b1/172.17.0.3:40827, sessionid=0x1003966037f0001 2024-11-13T13:48:05,392 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T13:48:05,392 DEBUG [RS:0;0b62828576b1:40827 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b62828576b1,40827,1731505683846 2024-11-13T13:48:05,392 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b62828576b1,40827,1731505683846' 2024-11-13T13:48:05,393 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T13:48:05,394 INFO [RS:1;0b62828576b1:36171 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:05,394 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T13:48:05,394 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(1482): Serving as 0b62828576b1,36171,1731505683989, RpcServer on 0b62828576b1/172.17.0.3:36171, sessionid=0x1003966037f0002 2024-11-13T13:48:05,395 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T13:48:05,395 DEBUG [RS:1;0b62828576b1:36171 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b62828576b1,36171,1731505683989 2024-11-13T13:48:05,395 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b62828576b1,36171,1731505683989' 2024-11-13T13:48:05,395 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T13:48:05,395 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T13:48:05,395 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T13:48:05,395 DEBUG [RS:0;0b62828576b1:40827 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b62828576b1,40827,1731505683846 2024-11-13T13:48:05,395 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b62828576b1,40827,1731505683846' 2024-11-13T13:48:05,395 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T13:48:05,396 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T13:48:05,396 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T13:48:05,397 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T13:48:05,397 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T13:48:05,397 DEBUG [RS:1;0b62828576b1:36171 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b62828576b1,36171,1731505683989 2024-11-13T13:48:05,397 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b62828576b1,36171,1731505683989' 2024-11-13T13:48:05,397 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T13:48:05,397 DEBUG [RS:0;0b62828576b1:40827 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T13:48:05,397 INFO [RS:0;0b62828576b1:40827 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T13:48:05,397 INFO [RS:0;0b62828576b1:40827 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T13:48:05,398 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T13:48:05,399 DEBUG [RS:1;0b62828576b1:36171 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T13:48:05,399 INFO [RS:1;0b62828576b1:36171 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T13:48:05,399 INFO [RS:1;0b62828576b1:36171 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T13:48:05,503 INFO [RS:0;0b62828576b1:40827 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T13:48:05,503 INFO [RS:1;0b62828576b1:36171 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T13:48:05,504 INFO [RS:0;0b62828576b1:40827 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0b62828576b1%2C40827%2C1731505683846, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,40827,1731505683846, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs, maxLogs=55 2024-11-13T13:48:05,505 INFO [RS:1;0b62828576b1:36171 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0b62828576b1%2C36171%2C1731505683989, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,36171,1731505683989, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs, maxLogs=55 2024-11-13T13:48:05,529 INFO [RS:0;0b62828576b1:40827 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,40827,1731505683846/0b62828576b1%2C40827%2C1731505683846.1731505685509 2024-11-13T13:48:05,529 INFO [RS:1;0b62828576b1:36171 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,36171,1731505683989/0b62828576b1%2C36171%2C1731505683989.1731505685509 2024-11-13T13:48:05,530 DEBUG [RS:1;0b62828576b1:36171 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T13:48:05,530 DEBUG [RS:0;0b62828576b1:40827 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T13:48:05,592 DEBUG [0b62828576b1:40603 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T13:48:05,604 DEBUG [0b62828576b1:40603 {}] balancer.BalancerClusterState(204): Hosts are {0b62828576b1=0} racks are {/default-rack=0} 2024-11-13T13:48:05,608 DEBUG [0b62828576b1:40603 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T13:48:05,608 DEBUG [0b62828576b1:40603 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T13:48:05,608 DEBUG [0b62828576b1:40603 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T13:48:05,608 DEBUG [0b62828576b1:40603 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T13:48:05,608 INFO [0b62828576b1:40603 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T13:48:05,608 INFO [0b62828576b1:40603 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T13:48:05,609 DEBUG [0b62828576b1:40603 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T13:48:05,616 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b62828576b1,36171,1731505683989 2024-11-13T13:48:05,622 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b62828576b1,36171,1731505683989, state=OPENING 2024-11-13T13:48:05,628 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T13:48:05,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:05,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:05,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:05,631 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:05,631 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:05,631 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:05,633 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T13:48:05,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b62828576b1,36171,1731505683989}] 2024-11-13T13:48:05,809 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T13:48:05,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58411, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T13:48:05,822 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T13:48:05,822 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T13:48:05,823 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T13:48:05,824 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0b62828576b1%2C36171%2C1731505683989.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,36171,1731505683989, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs, maxLogs=55 2024-11-13T13:48:05,845 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,36171,1731505683989/0b62828576b1%2C36171%2C1731505683989.meta.1731505685825.meta 2024-11-13T13:48:05,845 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T13:48:05,846 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T13:48:05,847 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T13:48:05,850 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T13:48:05,855 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T13:48:05,859 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T13:48:05,860 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:05,860 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T13:48:05,860 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T13:48:05,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T13:48:05,863 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T13:48:05,864 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:05,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:05,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T13:48:05,865 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T13:48:05,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:05,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:05,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T13:48:05,867 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T13:48:05,867 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:05,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:05,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T13:48:05,868 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T13:48:05,868 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:05,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:05,869 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T13:48:05,870 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740 2024-11-13T13:48:05,871 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740 2024-11-13T13:48:05,871 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T13:48:05,871 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T13:48:05,872 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T13:48:05,873 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T13:48:05,873 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73317513, jitterRate=0.09251607954502106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T13:48:05,874 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T13:48:05,875 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731505685861Writing region info on filesystem at 1731505685861Initializing all the Stores at 1731505685861Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505685861Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505685862 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505685862Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505685862Cleaning up temporary data from old regions at 1731505685871 (+9 ms)Running coprocessor post-open hooks at 1731505685874 (+3 ms)Region opened successfully at 1731505685875 (+1 ms) 2024-11-13T13:48:05,882 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731505685800 2024-11-13T13:48:05,894 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T13:48:05,895 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T13:48:05,896 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0b62828576b1,36171,1731505683989 2024-11-13T13:48:05,897 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b62828576b1,36171,1731505683989, state=OPEN 2024-11-13T13:48:05,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T13:48:05,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T13:48:05,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T13:48:05,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:05,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:05,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:05,901 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b62828576b1,36171,1731505683989 2024-11-13T13:48:05,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T13:48:05,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b62828576b1,36171,1731505683989 in 266 msec 2024-11-13T13:48:05,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T13:48:05,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 732 msec 2024-11-13T13:48:05,910 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T13:48:05,910 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T13:48:05,933 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T13:48:05,934 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b62828576b1,36171,1731505683989, seqNum=-1] 2024-11-13T13:48:05,956 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T13:48:05,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51295, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T13:48:05,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0430 sec 2024-11-13T13:48:05,978 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731505685978, completionTime=-1 2024-11-13T13:48:05,982 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-13T13:48:05,983 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T13:48:06,011 INFO [master/0b62828576b1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T13:48:06,011 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731505746011 2024-11-13T13:48:06,011 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731505806011 2024-11-13T13:48:06,011 INFO [master/0b62828576b1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-13T13:48:06,013 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T13:48:06,020 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,40603,1731505682995-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:06,021 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,40603,1731505682995-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:06,021 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,40603,1731505682995-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:06,023 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0b62828576b1:40603, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:06,023 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:06,024 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:06,029 DEBUG [master/0b62828576b1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T13:48:06,070 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.971sec 2024-11-13T13:48:06,075 INFO [master/0b62828576b1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T13:48:06,077 INFO [master/0b62828576b1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T13:48:06,079 INFO [master/0b62828576b1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T13:48:06,080 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T13:48:06,080 INFO [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T13:48:06,081 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,40603,1731505682995-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T13:48:06,081 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,40603,1731505682995-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T13:48:06,082 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T13:48:06,083 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T13:48:06,084 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,40603,1731505682995-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:06,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T13:48:06,138 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T13:48:06,139 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T13:48:06,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b62828576b1,40603,-1 for getting cluster id 2024-11-13T13:48:06,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T13:48:06,154 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b6f7d11-e80f-4ae3-a253-a3c3fe358ca7' 2024-11-13T13:48:06,156 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T13:48:06,156 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b6f7d11-e80f-4ae3-a253-a3c3fe358ca7" 2024-11-13T13:48:06,157 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24202fb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T13:48:06,157 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b62828576b1,40603,-1] 2024-11-13T13:48:06,160 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T13:48:06,161 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:06,163 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T13:48:06,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T13:48:06,168 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T13:48:06,175 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b62828576b1,36171,1731505683989, seqNum=-1] 2024-11-13T13:48:06,176 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T13:48:06,179 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T13:48:06,198 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0b62828576b1,40603,1731505682995 2024-11-13T13:48:06,203 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T13:48:06,207 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b62828576b1,40603,1731505682995 2024-11-13T13:48:06,210 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2c4204b9 2024-11-13T13:48:06,211 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T13:48:06,214 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T13:48:06,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testBlockEvictionOnRegionMove1731505686199', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T13:48:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731505686199 2024-11-13T13:48:06,230 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731505686199 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T13:48:06,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731505686199" procId is: 4 2024-11-13T13:48:06,233 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:06,234 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731505686199 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T13:48:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T13:48:06,249 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731505686199-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 323c2bf4df32e563cbb9ff78f7a21f62, NAME => 'testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731505686199', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2 2024-11-13T13:48:06,256 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731505686199-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:06,257 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731505686199-pool-0 {}] regionserver.HRegion(1722): Closing 323c2bf4df32e563cbb9ff78f7a21f62, disabling compactions & flushes 2024-11-13T13:48:06,257 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731505686199-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:06,257 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731505686199-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:06,257 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731505686199-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. after waiting 0 ms 2024-11-13T13:48:06,257 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731505686199-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:06,257 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731505686199-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:06,257 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731505686199-pool-0 {}] regionserver.HRegion(1676): Region close journal for 323c2bf4df32e563cbb9ff78f7a21f62: Waiting for close lock at 1731505686256Disabling compacts and flushes for region at 1731505686256Disabling writes for close at 1731505686257 (+1 ms)Writing region close event to WAL at 1731505686257Closed at 1731505686257 2024-11-13T13:48:06,259 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731505686199 execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T13:48:06,264 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731505686259"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731505686259"}]},"ts":"1731505686259"} 2024-11-13T13:48:06,269 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T13:48:06,271 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731505686199 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T13:48:06,273 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731505686199","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731505686271"}]},"ts":"1731505686271"} 2024-11-13T13:48:06,278 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731505686199, state=ENABLING in hbase:meta 2024-11-13T13:48:06,279 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0b62828576b1=0} racks are {/default-rack=0} 2024-11-13T13:48:06,280 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T13:48:06,280 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T13:48:06,280 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T13:48:06,280 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T13:48:06,280 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T13:48:06,280 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T13:48:06,280 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T13:48:06,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731505686199, region=323c2bf4df32e563cbb9ff78f7a21f62, ASSIGN}] 2024-11-13T13:48:06,283 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731505686199, region=323c2bf4df32e563cbb9ff78f7a21f62, ASSIGN 2024-11-13T13:48:06,284 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731505686199, region=323c2bf4df32e563cbb9ff78f7a21f62, ASSIGN; state=OFFLINE, location=0b62828576b1,40827,1731505683846; forceNewPlan=false, retain=false 2024-11-13T13:48:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T13:48:06,437 INFO [0b62828576b1:40603 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T13:48:06,438 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=323c2bf4df32e563cbb9ff78f7a21f62, regionState=OPENING, regionLocation=0b62828576b1,40827,1731505683846 2024-11-13T13:48:06,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731505686199, region=323c2bf4df32e563cbb9ff78f7a21f62, ASSIGN because future has completed 2024-11-13T13:48:06,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 323c2bf4df32e563cbb9ff78f7a21f62, server=0b62828576b1,40827,1731505683846}] 2024-11-13T13:48:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T13:48:06,597 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T13:48:06,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42669, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T13:48:06,602 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:06,602 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 323c2bf4df32e563cbb9ff78f7a21f62, NAME => 'testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62.', STARTKEY => '', ENDKEY => ''} 2024-11-13T13:48:06,603 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731505686199 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:06,603 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:06,603 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:06,603 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:06,605 INFO [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:06,609 INFO [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 323c2bf4df32e563cbb9ff78f7a21f62 columnFamilyName family 2024-11-13T13:48:06,609 DEBUG [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:06,610 INFO [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] regionserver.HStore(327): Store=323c2bf4df32e563cbb9ff78f7a21f62/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T13:48:06,610 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:06,611 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:06,611 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:06,611 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:06,611 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:06,612 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:06,619 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T13:48:06,619 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 323c2bf4df32e563cbb9ff78f7a21f62; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66249683, jitterRate=-0.012802794575691223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T13:48:06,619 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:06,620 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 323c2bf4df32e563cbb9ff78f7a21f62: Running coprocessor pre-open hook at 1731505686604Writing region info on filesystem at 1731505686604Initializing all the Stores at 1731505686604Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505686604Cleaning up temporary data from old regions at 1731505686611 (+7 ms)Running coprocessor post-open hooks at 1731505686619 (+8 ms)Region opened successfully at 1731505686620 (+1 ms) 2024-11-13T13:48:06,621 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62., pid=6, masterSystemTime=1731505686596 2024-11-13T13:48:06,625 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:06,625 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:06,625 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=323c2bf4df32e563cbb9ff78f7a21f62, regionState=OPEN, openSeqNum=2, regionLocation=0b62828576b1,40827,1731505683846 2024-11-13T13:48:06,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 323c2bf4df32e563cbb9ff78f7a21f62, server=0b62828576b1,40827,1731505683846 because future has completed 2024-11-13T13:48:06,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T13:48:06,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 323c2bf4df32e563cbb9ff78f7a21f62, server=0b62828576b1,40827,1731505683846 in 187 msec 2024-11-13T13:48:06,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T13:48:06,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731505686199, region=323c2bf4df32e563cbb9ff78f7a21f62, ASSIGN in 351 msec 2024-11-13T13:48:06,636 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731505686199 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T13:48:06,636 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731505686199","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731505686636"}]},"ts":"1731505686636"} 2024-11-13T13:48:06,640 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731505686199, state=ENABLED in hbase:meta 2024-11-13T13:48:06,641 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731505686199 execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T13:48:06,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731505686199 in 418 msec 2024-11-13T13:48:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T13:48:06,873 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731505686199 get assigned. Timeout = 60000ms 2024-11-13T13:48:06,873 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731505686199 completed 2024-11-13T13:48:06,876 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T13:48:06,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731505686199 assigned to meta. Checking AM states. 2024-11-13T13:48:06,883 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T13:48:06,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731505686199 assigned. 2024-11-13T13:48:06,893 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731505686199', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62., hostname=0b62828576b1,40827,1731505683846, seqNum=2] 2024-11-13T13:48:06,894 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T13:48:06,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T13:48:06,907 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 323c2bf4df32e563cbb9ff78f7a21f62 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-13T13:48:06,934 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7e5244987f8b4aae812e0cb245d429cb 2024-11-13T13:48:06,973 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/.tmp/family/7e5244987f8b4aae812e0cb245d429cb is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-13T13:48:06,979 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/.tmp/family/7e5244987f8b4aae812e0cb245d429cb 2024-11-13T13:48:07,018 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/.tmp/family/7e5244987f8b4aae812e0cb245d429cb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/family/7e5244987f8b4aae812e0cb245d429cb 2024-11-13T13:48:07,020 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/family/7e5244987f8b4aae812e0cb245d429cb, delay=913 ms 2024-11-13T13:48:07,022 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/family/7e5244987f8b4aae812e0cb245d429cb, entries=2, sequenceid=6, filesize=4.9 K 2024-11-13T13:48:07,028 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 323c2bf4df32e563cbb9ff78f7a21f62 in 120ms, sequenceid=6, compaction requested=false 2024-11-13T13:48:07,029 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731505686199' 2024-11-13T13:48:07,031 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 323c2bf4df32e563cbb9ff78f7a21f62: 2024-11-13T13:48:07,935 DEBUG [hfile-prefetch-1731505687020 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 7e5244987f8b4aae812e0cb245d429cb_0. 2024-11-13T13:48:07,940 DEBUG [hfile-prefetch-1731505687020 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 7e5244987f8b4aae812e0cb245d429cb_117. 2024-11-13T13:48:07,941 DEBUG [hfile-prefetch-1731505687020 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/family/7e5244987f8b4aae812e0cb245d429cb, with total blocks 2, and data blocks 1 2024-11-13T13:48:07,941 DEBUG [hfile-prefetch-1731505687020 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-13T13:48:07,941 DEBUG [hfile-prefetch-1731505687020 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/family/7e5244987f8b4aae812e0cb245d429cb, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-13T13:48:07,942 DEBUG [hfile-prefetch-1731505687020 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/family/7e5244987f8b4aae812e0cb245d429cb. Will sleep 100ms and try the verification again. 2024-11-13T13:48:08,008 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-13T13:48:08,008 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-13T13:48:08,009 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T13:48:08,009 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T13:48:08,027 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T13:48:08,027 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731505688009 2024-11-13T13:48:08,027 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-13T13:48:08,042 DEBUG [hfile-prefetch-1731505687020 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/family/7e5244987f8b4aae812e0cb245d429cb, with total blocks 2, and data blocks 1 2024-11-13T13:48:08,042 DEBUG [hfile-prefetch-1731505687020 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-13T13:48:08,042 DEBUG [hfile-prefetch-1731505687020 {}] bucket.BucketCache(2294): found block for file 7e5244987f8b4aae812e0cb245d429cb in the backing map. Acquiring read lock for offset 0 2024-11-13T13:48:08,042 DEBUG [hfile-prefetch-1731505687020 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/family/7e5244987f8b4aae812e0cb245d429cb has now been fully cached. 2024-11-13T13:48:08,043 DEBUG [hfile-prefetch-1731505687020 {}] hfile.PrefetchExecutor(121): Prefetch completed for 7e5244987f8b4aae812e0cb245d429cb 2024-11-13T13:48:08,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=323c2bf4df32e563cbb9ff78f7a21f62, source=0b62828576b1,40827,1731505683846, destination=0b62828576b1,36171,1731505683989, warming up region on 0b62828576b1,36171,1731505683989 2024-11-13T13:48:08,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=323c2bf4df32e563cbb9ff78f7a21f62, source=0b62828576b1,40827,1731505683846, destination=0b62828576b1,36171,1731505683989, running balancer 2024-11-13T13:48:08,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731505686199, region=323c2bf4df32e563cbb9ff78f7a21f62, REOPEN/MOVE 2024-11-13T13:48:08,047 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731505686199, region=323c2bf4df32e563cbb9ff78f7a21f62, REOPEN/MOVE 2024-11-13T13:48:08,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36171 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36171 {}] regionserver.HRegion(7855): Warmup {ENCODED => 323c2bf4df32e563cbb9ff78f7a21f62, NAME => 'testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62.', STARTKEY => '', ENDKEY => ''} 2024-11-13T13:48:08,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36171 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:08,049 INFO [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,049 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=323c2bf4df32e563cbb9ff78f7a21f62, regionState=CLOSING, regionLocation=0b62828576b1,40827,1731505683846 2024-11-13T13:48:08,050 INFO [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 323c2bf4df32e563cbb9ff78f7a21f62 columnFamilyName family 2024-11-13T13:48:08,050 DEBUG [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:08,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731505686199, region=323c2bf4df32e563cbb9ff78f7a21f62, REOPEN/MOVE because future has completed 2024-11-13T13:48:08,054 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-13T13:48:08,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 323c2bf4df32e563cbb9ff78f7a21f62, server=0b62828576b1,40827,1731505683846}] 2024-11-13T13:48:08,056 DEBUG [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/family/7e5244987f8b4aae812e0cb245d429cb 2024-11-13T13:48:08,056 INFO [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] regionserver.HStore(327): Store=323c2bf4df32e563cbb9ff78f7a21f62/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T13:48:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36171 {}] regionserver.HRegion(1722): Closing 323c2bf4df32e563cbb9ff78f7a21f62, disabling compactions & flushes 2024-11-13T13:48:08,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36171 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36171 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36171 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. after waiting 0 ms 2024-11-13T13:48:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36171 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36171 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36171 {}] regionserver.HRegion(1676): Region close journal for 323c2bf4df32e563cbb9ff78f7a21f62: Waiting for close lock at 1731505688056Disabling compacts and flushes for region at 1731505688056Disabling writes for close at 1731505688057 (+1 ms)Writing region close event to WAL at 1731505688058 (+1 ms)Closed at 1731505688058 2024-11-13T13:48:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-13T13:48:08,214 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,214 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-13T13:48:08,215 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 323c2bf4df32e563cbb9ff78f7a21f62, disabling compactions & flushes 2024-11-13T13:48:08,215 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,215 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,215 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. after waiting 0 ms 2024-11-13T13:48:08,215 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,217 DEBUG [StoreFileCloser-323c2bf4df32e563cbb9ff78f7a21f62-family-1 {}] bucket.BucketCache(769): Removing file 7e5244987f8b4aae812e0cb245d429cb for region 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,217 DEBUG [StoreFileCloser-323c2bf4df32e563cbb9ff78f7a21f62-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file 7e5244987f8b4aae812e0cb245d429cb, starting offset: 0, end offset: 9223372036854775807 2024-11-13T13:48:08,218 DEBUG [StoreFileCloser-323c2bf4df32e563cbb9ff78f7a21f62-family-1 {}] bucket.BucketCache(857): removed key 7e5244987f8b4aae812e0cb245d429cb_0 from back map with offset lock 2094080 in the evict process 2024-11-13T13:48:08,225 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-13T13:48:08,226 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,227 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 323c2bf4df32e563cbb9ff78f7a21f62: Waiting for close lock at 1731505688215Running coprocessor pre-close hooks at 1731505688215Disabling compacts and flushes for region at 1731505688215Disabling writes for close at 1731505688215Writing region close event to WAL at 1731505688219 (+4 ms)Running coprocessor post-close hooks at 1731505688225 (+6 ms)Closed at 1731505688226 (+1 ms) 2024-11-13T13:48:08,227 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 323c2bf4df32e563cbb9ff78f7a21f62 move to 0b62828576b1,36171,1731505683989 record at close sequenceid=6 2024-11-13T13:48:08,227 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-13T13:48:08,227 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-13T13:48:08,228 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T13:48:08,228 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T13:48:08,231 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,232 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T13:48:08,232 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731505688227 2024-11-13T13:48:08,232 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-13T13:48:08,232 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=323c2bf4df32e563cbb9ff78f7a21f62, regionState=CLOSED 2024-11-13T13:48:08,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 323c2bf4df32e563cbb9ff78f7a21f62, server=0b62828576b1,40827,1731505683846 because future has completed 2024-11-13T13:48:08,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T13:48:08,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 323c2bf4df32e563cbb9ff78f7a21f62, server=0b62828576b1,40827,1731505683846 in 182 msec 2024-11-13T13:48:08,239 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731505686199, region=323c2bf4df32e563cbb9ff78f7a21f62, REOPEN/MOVE; state=CLOSED, location=0b62828576b1,36171,1731505683989; forceNewPlan=false, retain=false 2024-11-13T13:48:08,390 INFO [0b62828576b1:40603 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T13:48:08,391 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=323c2bf4df32e563cbb9ff78f7a21f62, regionState=OPENING, regionLocation=0b62828576b1,36171,1731505683989 2024-11-13T13:48:08,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731505686199, region=323c2bf4df32e563cbb9ff78f7a21f62, REOPEN/MOVE because future has completed 2024-11-13T13:48:08,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 323c2bf4df32e563cbb9ff78f7a21f62, server=0b62828576b1,36171,1731505683989}] 2024-11-13T13:48:08,550 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,550 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 323c2bf4df32e563cbb9ff78f7a21f62, NAME => 'testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62.', STARTKEY => '', ENDKEY => ''} 2024-11-13T13:48:08,551 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731505686199 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,551 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:08,551 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,551 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,552 INFO [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,554 INFO [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 323c2bf4df32e563cbb9ff78f7a21f62 columnFamilyName family 2024-11-13T13:48:08,554 DEBUG [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:08,557 DEBUG [StoreFileOpener-323c2bf4df32e563cbb9ff78f7a21f62-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/family/7e5244987f8b4aae812e0cb245d429cb, delay=933 ms 2024-11-13T13:48:08,558 DEBUG [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/family/7e5244987f8b4aae812e0cb245d429cb 2024-11-13T13:48:08,558 INFO [StoreOpener-323c2bf4df32e563cbb9ff78f7a21f62-1 {}] regionserver.HStore(327): Store=323c2bf4df32e563cbb9ff78f7a21f62/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T13:48:08,558 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,559 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,560 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,560 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,560 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,565 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,565 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 323c2bf4df32e563cbb9ff78f7a21f62; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61768478, jitterRate=-0.07957795262336731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T13:48:08,565 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,567 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 323c2bf4df32e563cbb9ff78f7a21f62: Running coprocessor pre-open hook at 1731505688551Writing region info on filesystem at 1731505688551Initializing all the Stores at 1731505688551Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505688552 (+1 ms)Cleaning up temporary data from old regions at 1731505688560 (+8 ms)Running coprocessor post-open hooks at 1731505688565 (+5 ms)Region opened successfully at 1731505688567 (+2 ms) 2024-11-13T13:48:08,568 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62., pid=9, masterSystemTime=1731505688547 2024-11-13T13:48:08,570 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,570 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,571 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=323c2bf4df32e563cbb9ff78f7a21f62, regionState=OPEN, openSeqNum=10, regionLocation=0b62828576b1,36171,1731505683989 2024-11-13T13:48:08,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 323c2bf4df32e563cbb9ff78f7a21f62, server=0b62828576b1,36171,1731505683989 because future has completed 2024-11-13T13:48:08,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-13T13:48:08,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 323c2bf4df32e563cbb9ff78f7a21f62, server=0b62828576b1,36171,1731505683989 in 180 msec 2024-11-13T13:48:08,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731505686199, region=323c2bf4df32e563cbb9ff78f7a21f62, REOPEN/MOVE in 530 msec 2024-11-13T13:48:08,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T13:48:08,601 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T13:48:08,601 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T13:48:08,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:08,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:08,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T13:48:08,607 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T13:48:08,607 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-13T13:48:08,607 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0b62828576b1,40603,1731505682995 2024-11-13T13:48:08,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T13:48:08,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T13:48:08,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:08,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:08,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T13:48:08,610 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T13:48:08,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:08,611 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T13:48:08,611 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T13:48:08,611 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T13:48:08,611 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T13:48:08,611 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T13:48:08,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:08,612 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b62828576b1,40827,1731505683846' ***** 2024-11-13T13:48:08,612 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T13:48:08,612 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b62828576b1,36171,1731505683989' ***** 2024-11-13T13:48:08,612 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T13:48:08,612 INFO [RS:0;0b62828576b1:40827 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T13:48:08,612 INFO [RS:1;0b62828576b1:36171 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T13:48:08,613 INFO [RS:1;0b62828576b1:36171 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/bucket.persistence 2024-11-13T13:48:08,613 INFO [RS:0;0b62828576b1:40827 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/bucket.persistence 2024-11-13T13:48:08,613 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T13:48:08,613 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T13:48:08,613 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T13:48:08,613 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T13:48:08,613 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T13:48:08,613 INFO [RS:1;0b62828576b1:36171 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T13:48:08,613 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T13:48:08,616 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T13:48:08,617 INFO [RS:0;0b62828576b1:40827 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T13:48:08,617 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T13:48:08,623 DEBUG [RS:1;0b62828576b1:36171 {}] bucket.BucketCache(1488): Thread RS:1;0b62828576b1:36171 started persisting bucket cache to file 2024-11-13T13:48:08,624 DEBUG [RS:1;0b62828576b1:36171 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T13:48:08,624 DEBUG [RS:1;0b62828576b1:36171 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T13:48:08,627 DEBUG [RS:0;0b62828576b1:40827 {}] bucket.BucketCache(1488): Thread RS:0;0b62828576b1:40827 started persisting bucket cache to file 2024-11-13T13:48:08,627 DEBUG [RS:0;0b62828576b1:40827 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T13:48:08,627 DEBUG [RS:0;0b62828576b1:40827 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T13:48:08,627 DEBUG [RS:1;0b62828576b1:36171 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T13:48:08,627 DEBUG [RS:1;0b62828576b1:36171 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731505688623 2024-11-13T13:48:08,627 DEBUG [RS:1;0b62828576b1:36171 {}] bucket.BucketCache(1510): Thread RS:1;0b62828576b1:36171 finished persisting bucket cache to file, renaming 2024-11-13T13:48:08,628 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T13:48:08,628 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T13:48:08,628 INFO [RS:1;0b62828576b1:36171 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T13:48:08,628 INFO [RS:1;0b62828576b1:36171 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T13:48:08,628 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(3091): Received CLOSE for 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,629 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(959): stopping server 0b62828576b1,36171,1731505683989 2024-11-13T13:48:08,629 INFO [RS:1;0b62828576b1:36171 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T13:48:08,629 INFO [RS:1;0b62828576b1:36171 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0b62828576b1:36171. 2024-11-13T13:48:08,629 DEBUG [RS:1;0b62828576b1:36171 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T13:48:08,629 DEBUG [RS:1;0b62828576b1:36171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:08,629 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T13:48:08,629 INFO [RS:1;0b62828576b1:36171 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T13:48:08,629 INFO [RS:1;0b62828576b1:36171 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T13:48:08,630 INFO [RS:1;0b62828576b1:36171 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T13:48:08,630 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T13:48:08,630 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 323c2bf4df32e563cbb9ff78f7a21f62, disabling compactions & flushes 2024-11-13T13:48:08,630 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,630 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,630 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. after waiting 0 ms 2024-11-13T13:48:08,630 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,630 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T13:48:08,630 DEBUG [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(1325): Online Regions={323c2bf4df32e563cbb9ff78f7a21f62=testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62., 1588230740=hbase:meta,,1.1588230740} 2024-11-13T13:48:08,630 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T13:48:08,630 DEBUG [RS:0;0b62828576b1:40827 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T13:48:08,631 DEBUG [RS:0;0b62828576b1:40827 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731505688627 2024-11-13T13:48:08,631 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T13:48:08,631 DEBUG [RS:0;0b62828576b1:40827 {}] bucket.BucketCache(1510): Thread RS:0;0b62828576b1:40827 finished persisting bucket cache to file, renaming 2024-11-13T13:48:08,631 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T13:48:08,631 DEBUG [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 323c2bf4df32e563cbb9ff78f7a21f62 2024-11-13T13:48:08,631 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T13:48:08,631 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T13:48:08,631 INFO [RS:0;0b62828576b1:40827 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T13:48:08,631 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T13:48:08,631 INFO [RS:0;0b62828576b1:40827 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T13:48:08,631 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T13:48:08,631 INFO [RS:0;0b62828576b1:40827 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T13:48:08,631 INFO [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(959): stopping server 0b62828576b1,40827,1731505683846 2024-11-13T13:48:08,631 INFO [RS:0;0b62828576b1:40827 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T13:48:08,631 INFO [RS:0;0b62828576b1:40827 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0b62828576b1:40827. 2024-11-13T13:48:08,631 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-13T13:48:08,631 DEBUG [RS:0;0b62828576b1:40827 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T13:48:08,631 DEBUG [RS:0;0b62828576b1:40827 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:08,631 DEBUG [StoreFileCloser-323c2bf4df32e563cbb9ff78f7a21f62-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/family/7e5244987f8b4aae812e0cb245d429cb 2024-11-13T13:48:08,632 INFO [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(976): stopping server 0b62828576b1,40827,1731505683846; all regions closed. 2024-11-13T13:48:08,637 DEBUG [RS:0;0b62828576b1:40827 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs 2024-11-13T13:48:08,637 INFO [RS:0;0b62828576b1:40827 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b62828576b1%2C40827%2C1731505683846:(num 1731505685509) 2024-11-13T13:48:08,637 DEBUG [RS:0;0b62828576b1:40827 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:08,637 INFO [RS:0;0b62828576b1:40827 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T13:48:08,637 INFO [RS:0;0b62828576b1:40827 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T13:48:08,638 INFO [RS:0;0b62828576b1:40827 {}] hbase.ChoreService(370): Chore service for: regionserver/0b62828576b1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T13:48:08,638 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnRegionMove1731505686199/323c2bf4df32e563cbb9ff78f7a21f62/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-13T13:48:08,638 INFO [RS:0;0b62828576b1:40827 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T13:48:08,638 INFO [RS:0;0b62828576b1:40827 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T13:48:08,638 INFO [regionserver/0b62828576b1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T13:48:08,638 INFO [RS:0;0b62828576b1:40827 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T13:48:08,638 INFO [RS:0;0b62828576b1:40827 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T13:48:08,638 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,638 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 323c2bf4df32e563cbb9ff78f7a21f62: Waiting for close lock at 1731505688629Running coprocessor pre-close hooks at 1731505688630 (+1 ms)Disabling compacts and flushes for region at 1731505688630Disabling writes for close at 1731505688630Writing region close event to WAL at 1731505688632 (+2 ms)Running coprocessor post-close hooks at 1731505688638 (+6 ms)Closed at 1731505688638 2024-11-13T13:48:08,638 INFO [RS:0;0b62828576b1:40827 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40827 2024-11-13T13:48:08,639 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62. 2024-11-13T13:48:08,639 INFO [regionserver/0b62828576b1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T13:48:08,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T13:48:08,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b62828576b1,40827,1731505683846 2024-11-13T13:48:08,643 INFO [RS:0;0b62828576b1:40827 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T13:48:08,644 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b62828576b1,40827,1731505683846] 2024-11-13T13:48:08,645 INFO [regionserver/0b62828576b1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T13:48:08,646 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b62828576b1,40827,1731505683846 already deleted, retry=false 2024-11-13T13:48:08,646 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b62828576b1,40827,1731505683846 expired; onlineServers=1 2024-11-13T13:48:08,647 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 75fe03561bc0422aaa5c65303ed6714b 2024-11-13T13:48:08,666 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/info/75fe03561bc0422aaa5c65303ed6714b is 205, key is testBlockEvictionOnRegionMove1731505686199,,1731505686215.323c2bf4df32e563cbb9ff78f7a21f62./info:regioninfo/1731505688571/Put/seqid=0 2024-11-13T13:48:08,667 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/info/75fe03561bc0422aaa5c65303ed6714b 2024-11-13T13:48:08,678 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dab2992e2fb7416085dd4c698a2e6f5b 2024-11-13T13:48:08,693 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/ns/dab2992e2fb7416085dd4c698a2e6f5b is 43, key is default/ns:d/1731505685963/Put/seqid=0 2024-11-13T13:48:08,694 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/ns/dab2992e2fb7416085dd4c698a2e6f5b 2024-11-13T13:48:08,704 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5392d2a345df4bd99e71fedf209b80a2 2024-11-13T13:48:08,718 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/table/5392d2a345df4bd99e71fedf209b80a2 is 78, key is testBlockEvictionOnRegionMove1731505686199/table:state/1731505686636/Put/seqid=0 2024-11-13T13:48:08,719 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/table/5392d2a345df4bd99e71fedf209b80a2 2024-11-13T13:48:08,723 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/info/75fe03561bc0422aaa5c65303ed6714b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/75fe03561bc0422aaa5c65303ed6714b 2024-11-13T13:48:08,724 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/75fe03561bc0422aaa5c65303ed6714b, delay=991 ms 2024-11-13T13:48:08,725 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/75fe03561bc0422aaa5c65303ed6714b, entries=15, sequenceid=15, filesize=7.6 K 2024-11-13T13:48:08,726 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/ns/dab2992e2fb7416085dd4c698a2e6f5b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/ns/dab2992e2fb7416085dd4c698a2e6f5b 2024-11-13T13:48:08,728 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/ns/dab2992e2fb7416085dd4c698a2e6f5b, delay=974 ms 2024-11-13T13:48:08,730 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/ns/dab2992e2fb7416085dd4c698a2e6f5b, entries=2, sequenceid=15, filesize=5.0 K 2024-11-13T13:48:08,731 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/table/5392d2a345df4bd99e71fedf209b80a2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/table/5392d2a345df4bd99e71fedf209b80a2 2024-11-13T13:48:08,733 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/table/5392d2a345df4bd99e71fedf209b80a2, delay=962 ms 2024-11-13T13:48:08,734 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/table/5392d2a345df4bd99e71fedf209b80a2, entries=2, sequenceid=15, filesize=5.3 K 2024-11-13T13:48:08,736 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=15, compaction requested=false 2024-11-13T13:48:08,736 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T13:48:08,737 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/75fe03561bc0422aaa5c65303ed6714b 2024-11-13T13:48:08,738 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/ns/dab2992e2fb7416085dd4c698a2e6f5b 2024-11-13T13:48:08,738 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/table/5392d2a345df4bd99e71fedf209b80a2 2024-11-13T13:48:08,743 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-13T13:48:08,744 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T13:48:08,745 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T13:48:08,745 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731505688630Running coprocessor pre-close hooks at 1731505688630Disabling compacts and flushes for region at 1731505688630Disabling writes for close at 1731505688631 (+1 ms)Obtaining lock to block concurrent updates at 1731505688631Preparing flush snapshotting stores in 1588230740 at 1731505688631Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731505688632 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731505688632Flushing 1588230740/info: creating writer at 1731505688633 (+1 ms)Flushing 1588230740/info: appending metadata at 1731505688664 (+31 ms)Flushing 1588230740/info: closing flushed file at 1731505688664Flushing 1588230740/ns: creating writer at 1731505688671 (+7 ms)Flushing 1588230740/ns: appending metadata at 1731505688692 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731505688692Flushing 1588230740/table: creating writer at 1731505688697 (+5 ms)Flushing 1588230740/table: appending metadata at 1731505688718 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731505688718Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@763a228: reopening flushed file at 1731505688722 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b25b7e3: reopening flushed file at 1731505688726 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4829dd7d: reopening flushed file at 1731505688730 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=15, compaction requested=false at 1731505688736 (+6 ms)Writing region close event to WAL at 1731505688739 (+3 ms)Running coprocessor post-close hooks at 1731505688744 (+5 ms)Closed at 1731505688745 (+1 ms) 2024-11-13T13:48:08,745 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T13:48:08,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:08,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40827-0x1003966037f0001, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:08,746 INFO [RS:0;0b62828576b1:40827 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T13:48:08,747 INFO [RS:0;0b62828576b1:40827 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b62828576b1,40827,1731505683846; zookeeper connection closed. 2024-11-13T13:48:08,747 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6befd8f7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6befd8f7 2024-11-13T13:48:08,831 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(976): stopping server 0b62828576b1,36171,1731505683989; all regions closed. 2024-11-13T13:48:08,834 DEBUG [RS:1;0b62828576b1:36171 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs 2024-11-13T13:48:08,834 INFO [RS:1;0b62828576b1:36171 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b62828576b1%2C36171%2C1731505683989.meta:.meta(num 1731505685825) 2024-11-13T13:48:08,835 DEBUG [RS:1;0b62828576b1:36171 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs 2024-11-13T13:48:08,835 INFO [RS:1;0b62828576b1:36171 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b62828576b1%2C36171%2C1731505683989:(num 1731505685509) 2024-11-13T13:48:08,835 DEBUG [RS:1;0b62828576b1:36171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:08,835 INFO [RS:1;0b62828576b1:36171 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T13:48:08,836 INFO [RS:1;0b62828576b1:36171 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T13:48:08,836 INFO [RS:1;0b62828576b1:36171 {}] hbase.ChoreService(370): Chore service for: regionserver/0b62828576b1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T13:48:08,836 INFO [RS:1;0b62828576b1:36171 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T13:48:08,836 INFO [regionserver/0b62828576b1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T13:48:08,836 INFO [RS:1;0b62828576b1:36171 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36171 2024-11-13T13:48:08,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b62828576b1,36171,1731505683989 2024-11-13T13:48:08,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T13:48:08,839 INFO [RS:1;0b62828576b1:36171 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T13:48:08,841 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b62828576b1,36171,1731505683989] 2024-11-13T13:48:08,842 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b62828576b1,36171,1731505683989 already deleted, retry=false 2024-11-13T13:48:08,842 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b62828576b1,36171,1731505683989 expired; onlineServers=0 2024-11-13T13:48:08,843 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0b62828576b1,40603,1731505682995' ***** 2024-11-13T13:48:08,843 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T13:48:08,843 INFO [M:0;0b62828576b1:40603 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T13:48:08,843 INFO [M:0;0b62828576b1:40603 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T13:48:08,843 DEBUG [M:0;0b62828576b1:40603 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T13:48:08,843 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T13:48:08,843 DEBUG [M:0;0b62828576b1:40603 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T13:48:08,843 DEBUG [master/0b62828576b1:0:becomeActiveMaster-HFileCleaner.small.0-1731505685109 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b62828576b1:0:becomeActiveMaster-HFileCleaner.small.0-1731505685109,5,FailOnTimeoutGroup] 2024-11-13T13:48:08,843 DEBUG [master/0b62828576b1:0:becomeActiveMaster-HFileCleaner.large.0-1731505685100 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b62828576b1:0:becomeActiveMaster-HFileCleaner.large.0-1731505685100,5,FailOnTimeoutGroup] 2024-11-13T13:48:08,843 INFO [M:0;0b62828576b1:40603 {}] hbase.ChoreService(370): Chore service for: master/0b62828576b1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T13:48:08,843 INFO [M:0;0b62828576b1:40603 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T13:48:08,844 DEBUG [M:0;0b62828576b1:40603 {}] master.HMaster(1795): Stopping service threads 2024-11-13T13:48:08,844 INFO [M:0;0b62828576b1:40603 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T13:48:08,844 INFO [M:0;0b62828576b1:40603 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T13:48:08,844 INFO [M:0;0b62828576b1:40603 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T13:48:08,844 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T13:48:08,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T13:48:08,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:08,846 DEBUG [M:0;0b62828576b1:40603 {}] zookeeper.ZKUtil(347): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T13:48:08,846 WARN [M:0;0b62828576b1:40603 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T13:48:08,847 INFO [M:0;0b62828576b1:40603 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/.lastflushedseqids 2024-11-13T13:48:08,862 INFO [M:0;0b62828576b1:40603 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T13:48:08,862 INFO [M:0;0b62828576b1:40603 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T13:48:08,862 DEBUG [M:0;0b62828576b1:40603 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T13:48:08,863 INFO [M:0;0b62828576b1:40603 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:08,863 DEBUG [M:0;0b62828576b1:40603 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:08,863 DEBUG [M:0;0b62828576b1:40603 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T13:48:08,863 DEBUG [M:0;0b62828576b1:40603 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:08,863 INFO [M:0;0b62828576b1:40603 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-13T13:48:08,875 DEBUG [M:0;0b62828576b1:40603 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 775c49de20384454b24db1f1d31200f8 2024-11-13T13:48:08,896 DEBUG [M:0;0b62828576b1:40603 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/775c49de20384454b24db1f1d31200f8 is 82, key is hbase:meta,,1/info:regioninfo/1731505685895/Put/seqid=0 2024-11-13T13:48:08,898 INFO [M:0;0b62828576b1:40603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/775c49de20384454b24db1f1d31200f8 2024-11-13T13:48:08,908 DEBUG [M:0;0b62828576b1:40603 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3da5e100e5bc4b1b8076818de68f2318 2024-11-13T13:48:08,928 DEBUG [M:0;0b62828576b1:40603 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3da5e100e5bc4b1b8076818de68f2318 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731505686642/Put/seqid=0 2024-11-13T13:48:08,929 INFO [M:0;0b62828576b1:40603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3da5e100e5bc4b1b8076818de68f2318 2024-11-13T13:48:08,938 DEBUG [M:0;0b62828576b1:40603 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 71d99197d4e04c6bbd024a75c7f4f5fb 2024-11-13T13:48:08,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:08,941 INFO [RS:1;0b62828576b1:36171 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T13:48:08,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36171-0x1003966037f0002, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:08,941 INFO [RS:1;0b62828576b1:36171 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b62828576b1,36171,1731505683989; zookeeper connection closed. 2024-11-13T13:48:08,942 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@56344959 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@56344959 2024-11-13T13:48:08,942 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-13T13:48:08,952 DEBUG [M:0;0b62828576b1:40603 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/71d99197d4e04c6bbd024a75c7f4f5fb is 69, key is 0b62828576b1,36171,1731505683989/rs:state/1731505685243/Put/seqid=0 2024-11-13T13:48:08,953 INFO [M:0;0b62828576b1:40603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/71d99197d4e04c6bbd024a75c7f4f5fb 2024-11-13T13:48:08,956 DEBUG [M:0;0b62828576b1:40603 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/775c49de20384454b24db1f1d31200f8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/775c49de20384454b24db1f1d31200f8 2024-11-13T13:48:08,959 INFO [M:0;0b62828576b1:40603 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/775c49de20384454b24db1f1d31200f8, entries=8, sequenceid=84, filesize=5.5 K 2024-11-13T13:48:08,959 DEBUG [M:0;0b62828576b1:40603 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3da5e100e5bc4b1b8076818de68f2318 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3da5e100e5bc4b1b8076818de68f2318 2024-11-13T13:48:08,962 INFO [M:0;0b62828576b1:40603 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3da5e100e5bc4b1b8076818de68f2318, entries=9, sequenceid=84, filesize=6.5 K 2024-11-13T13:48:08,963 DEBUG [M:0;0b62828576b1:40603 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/71d99197d4e04c6bbd024a75c7f4f5fb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/71d99197d4e04c6bbd024a75c7f4f5fb 2024-11-13T13:48:08,966 INFO [M:0;0b62828576b1:40603 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/71d99197d4e04c6bbd024a75c7f4f5fb, entries=2, sequenceid=84, filesize=5.1 K 2024-11-13T13:48:08,967 INFO [M:0;0b62828576b1:40603 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 103ms, sequenceid=84, compaction requested=false 2024-11-13T13:48:08,969 INFO [M:0;0b62828576b1:40603 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:08,969 DEBUG [M:0;0b62828576b1:40603 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731505688862Disabling compacts and flushes for region at 1731505688862Disabling writes for close at 1731505688863 (+1 ms)Obtaining lock to block concurrent updates at 1731505688863Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731505688863Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1731505688864 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731505688864Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731505688864Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731505688896 (+32 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731505688896Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731505688901 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731505688927 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731505688927Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731505688931 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731505688951 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731505688952 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62c0e1df: reopening flushed file at 1731505688955 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b09b77d: reopening flushed file at 1731505688959 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77240343: reopening flushed file at 1731505688962 (+3 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 103ms, sequenceid=84, compaction requested=false at 1731505688967 (+5 ms)Writing region close event to WAL at 1731505688969 (+2 ms)Closed at 1731505688969 2024-11-13T13:48:08,971 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T13:48:08,971 INFO [M:0;0b62828576b1:40603 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T13:48:08,971 INFO [M:0;0b62828576b1:40603 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40603 2024-11-13T13:48:08,972 INFO [M:0;0b62828576b1:40603 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T13:48:09,074 INFO [M:0;0b62828576b1:40603 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T13:48:09,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:09,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x1003966037f0000, quorum=127.0.0.1:57932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:09,079 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T13:48:09,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T13:48:09,099 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:09,108 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/0b62828576b1:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731505688728 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/0b62828576b1:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731505688557 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/0b62828576b1:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/0b62828576b1:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731505688724 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731505687020 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=575 (was 616), ProcessCount=11 (was 11), AvailableMemoryMB=2795 (was 3131) 2024-11-13T13:48:09,114 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=575, ProcessCount=11, AvailableMemoryMB=2795 2024-11-13T13:48:09,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/hadoop-log-dir so I do NOT create it in target/test-data/627764bd-72cd-55ed-0777-7d327eec7752 2024-11-13T13:48:09,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/hadoop-tmp-dir so I do NOT create it in target/test-data/627764bd-72cd-55ed-0777-7d327eec7752 2024-11-13T13:48:09,115 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:09,118 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/cluster_d9b8a3d3-d0fc-0582-27f0-fbaa872b73b4, deleteOnExit=true 2024-11-13T13:48:09,121 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/cluster_d9b8a3d3-d0fc-0582-27f0-fbaa872b73b4/zookeeper_0, clientPort=58953, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/cluster_d9b8a3d3-d0fc-0582-27f0-fbaa872b73b4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/cluster_d9b8a3d3-d0fc-0582-27f0-fbaa872b73b4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T13:48:09,122 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58953 2024-11-13T13:48:09,135 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2 with version=8 2024-11-13T13:48:09,135 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/hbase-staging 2024-11-13T13:48:09,138 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0b62828576b1:0 server-side Connection retries=45 2024-11-13T13:48:09,138 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:09,138 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:09,138 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T13:48:09,138 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:09,138 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T13:48:09,139 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T13:48:09,139 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T13:48:09,139 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34663 2024-11-13T13:48:09,141 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34663 connecting to ZooKeeper ensemble=127.0.0.1:58953 2024-11-13T13:48:09,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:346630x0, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T13:48:09,148 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34663-0x10039661cbf0000 connected 2024-11-13T13:48:09,163 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:09,164 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:09,165 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T13:48:09,165 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2, hbase.cluster.distributed=false 2024-11-13T13:48:09,167 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T13:48:09,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34663 2024-11-13T13:48:09,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34663 2024-11-13T13:48:09,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34663 2024-11-13T13:48:09,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34663 2024-11-13T13:48:09,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34663 2024-11-13T13:48:09,185 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b62828576b1:0 server-side Connection retries=45 2024-11-13T13:48:09,186 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:09,186 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:09,186 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T13:48:09,186 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:09,186 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T13:48:09,186 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T13:48:09,186 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T13:48:09,187 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44471 2024-11-13T13:48:09,188 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44471 connecting to ZooKeeper ensemble=127.0.0.1:58953 2024-11-13T13:48:09,189 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:09,189 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:09,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444710x0, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T13:48:09,193 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44471-0x10039661cbf0001 connected 2024-11-13T13:48:09,193 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T13:48:09,193 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T13:48:09,194 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.cache 2024-11-13T13:48:09,194 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T13:48:09,194 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T13:48:09,194 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T13:48:09,195 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T13:48:09,195 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-13T13:48:09,195 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T13:48:09,195 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.persistence is complete. 2024-11-13T13:48:09,195 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T13:48:09,198 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T13:48:09,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T13:48:09,200 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T13:48:09,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44471 2024-11-13T13:48:09,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44471 2024-11-13T13:48:09,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44471 2024-11-13T13:48:09,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44471 2024-11-13T13:48:09,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44471 2024-11-13T13:48:09,219 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b62828576b1:0 server-side Connection retries=45 2024-11-13T13:48:09,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:09,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:09,219 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T13:48:09,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:09,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T13:48:09,220 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T13:48:09,220 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T13:48:09,221 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34919 2024-11-13T13:48:09,222 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34919 connecting to ZooKeeper ensemble=127.0.0.1:58953 2024-11-13T13:48:09,222 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:09,223 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:09,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:349190x0, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T13:48:09,227 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34919-0x10039661cbf0002 connected 2024-11-13T13:48:09,227 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T13:48:09,227 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T13:48:09,228 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.cache 2024-11-13T13:48:09,228 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T13:48:09,228 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T13:48:09,228 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T13:48:09,230 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T13:48:09,230 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-13T13:48:09,230 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T13:48:09,231 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T13:48:09,231 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.persistence is complete. 2024-11-13T13:48:09,231 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T13:48:09,232 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T13:48:09,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T13:48:09,237 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34919 2024-11-13T13:48:09,237 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34919 2024-11-13T13:48:09,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34919 2024-11-13T13:48:09,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34919 2024-11-13T13:48:09,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34919 2024-11-13T13:48:09,254 DEBUG [M:0;0b62828576b1:34663 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0b62828576b1:34663 2024-11-13T13:48:09,255 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0b62828576b1,34663,1731505689137 2024-11-13T13:48:09,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T13:48:09,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T13:48:09,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T13:48:09,258 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0b62828576b1,34663,1731505689137 2024-11-13T13:48:09,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T13:48:09,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T13:48:09,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,262 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T13:48:09,264 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0b62828576b1,34663,1731505689137 from backup master directory 2024-11-13T13:48:09,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0b62828576b1,34663,1731505689137 2024-11-13T13:48:09,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T13:48:09,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T13:48:09,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T13:48:09,266 WARN [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T13:48:09,266 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0b62828576b1,34663,1731505689137 2024-11-13T13:48:09,267 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/hbase.id] with ID: a8ec8e68-2e4c-43c8-95f8-f934aa5c759d 2024-11-13T13:48:09,267 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/.tmp/hbase.id 2024-11-13T13:48:09,272 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/hbase.id] 2024-11-13T13:48:09,314 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:09,314 INFO [master/0b62828576b1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T13:48:09,316 INFO [master/0b62828576b1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-13T13:48:09,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,324 INFO [master/0b62828576b1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T13:48:09,325 INFO [master/0b62828576b1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T13:48:09,326 INFO [master/0b62828576b1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T13:48:09,347 INFO [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store 2024-11-13T13:48:09,354 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:09,354 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T13:48:09,354 INFO [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:09,354 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:09,354 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T13:48:09,354 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:09,354 INFO [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:09,354 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731505689354Disabling compacts and flushes for region at 1731505689354Disabling writes for close at 1731505689354Writing region close event to WAL at 1731505689354Closed at 1731505689354 2024-11-13T13:48:09,354 WARN [master/0b62828576b1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/.initializing 2024-11-13T13:48:09,354 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/WALs/0b62828576b1,34663,1731505689137 2024-11-13T13:48:09,360 INFO [master/0b62828576b1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T13:48:09,363 INFO [master/0b62828576b1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b62828576b1%2C34663%2C1731505689137, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/WALs/0b62828576b1,34663,1731505689137, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/oldWALs, maxLogs=10 2024-11-13T13:48:09,382 INFO [master/0b62828576b1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/WALs/0b62828576b1,34663,1731505689137/0b62828576b1%2C34663%2C1731505689137.1731505689364 2024-11-13T13:48:09,383 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T13:48:09,383 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T13:48:09,383 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:09,383 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:09,383 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:09,384 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:09,388 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T13:48:09,389 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:09,389 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:09,389 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:09,393 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T13:48:09,393 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:09,393 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T13:48:09,393 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:09,397 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T13:48:09,397 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:09,397 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T13:48:09,397 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:09,400 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T13:48:09,401 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:09,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T13:48:09,401 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:09,401 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:09,401 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:09,402 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:09,402 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:09,402 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T13:48:09,402 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T13:48:09,408 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T13:48:09,409 INFO [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59313227, jitterRate=-0.11616404354572296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T13:48:09,409 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731505689383Initializing all the Stores at 1731505689384 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505689384Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505689384Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505689384Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505689384Cleaning up temporary data from old regions at 1731505689402 (+18 ms)Region opened successfully at 1731505689409 (+7 ms) 2024-11-13T13:48:09,410 INFO [master/0b62828576b1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T13:48:09,414 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74577707, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b62828576b1/172.17.0.3:0 2024-11-13T13:48:09,415 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T13:48:09,415 INFO [master/0b62828576b1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T13:48:09,415 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T13:48:09,415 INFO [master/0b62828576b1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T13:48:09,416 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T13:48:09,416 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-13T13:48:09,416 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T13:48:09,418 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T13:48:09,419 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T13:48:09,420 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T13:48:09,421 INFO [master/0b62828576b1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T13:48:09,421 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T13:48:09,423 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T13:48:09,423 INFO [master/0b62828576b1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T13:48:09,424 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T13:48:09,425 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T13:48:09,426 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T13:48:09,427 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T13:48:09,429 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T13:48:09,431 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T13:48:09,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T13:48:09,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T13:48:09,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T13:48:09,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,434 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0b62828576b1,34663,1731505689137, sessionid=0x10039661cbf0000, setting cluster-up flag (Was=false) 2024-11-13T13:48:09,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,442 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T13:48:09,443 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b62828576b1,34663,1731505689137 2024-11-13T13:48:09,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,452 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T13:48:09,454 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b62828576b1,34663,1731505689137 2024-11-13T13:48:09,454 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T13:48:09,456 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T13:48:09,456 INFO [master/0b62828576b1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T13:48:09,456 INFO [master/0b62828576b1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T13:48:09,456 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0b62828576b1,34663,1731505689137 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T13:48:09,458 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0b62828576b1:0, corePoolSize=5, maxPoolSize=5 2024-11-13T13:48:09,458 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0b62828576b1:0, corePoolSize=5, maxPoolSize=5 2024-11-13T13:48:09,458 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0b62828576b1:0, corePoolSize=5, maxPoolSize=5 2024-11-13T13:48:09,458 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0b62828576b1:0, corePoolSize=5, maxPoolSize=5 2024-11-13T13:48:09,458 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0b62828576b1:0, corePoolSize=10, maxPoolSize=10 2024-11-13T13:48:09,458 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,458 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0b62828576b1:0, corePoolSize=2, maxPoolSize=2 2024-11-13T13:48:09,459 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,459 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731505719459 2024-11-13T13:48:09,460 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T13:48:09,460 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T13:48:09,460 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T13:48:09,460 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T13:48:09,460 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T13:48:09,460 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T13:48:09,460 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,460 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T13:48:09,460 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T13:48:09,460 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T13:48:09,461 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T13:48:09,461 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T13:48:09,461 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:09,461 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T13:48:09,462 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T13:48:09,462 INFO [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T13:48:09,465 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0b62828576b1:0:becomeActiveMaster-HFileCleaner.large.0-1731505689462,5,FailOnTimeoutGroup] 2024-11-13T13:48:09,471 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0b62828576b1:0:becomeActiveMaster-HFileCleaner.small.0-1731505689465,5,FailOnTimeoutGroup] 2024-11-13T13:48:09,471 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,471 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T13:48:09,471 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,472 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,483 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T13:48:09,483 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2 2024-11-13T13:48:09,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:09,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T13:48:09,494 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T13:48:09,494 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:09,495 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:09,495 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T13:48:09,498 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T13:48:09,498 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:09,498 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:09,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T13:48:09,502 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T13:48:09,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:09,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:09,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T13:48:09,506 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T13:48:09,506 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:09,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:09,506 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T13:48:09,507 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740 2024-11-13T13:48:09,507 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740 2024-11-13T13:48:09,507 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T13:48:09,507 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T13:48:09,508 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T13:48:09,508 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T13:48:09,514 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T13:48:09,515 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59923946, jitterRate=-0.10706362128257751}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T13:48:09,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731505689490Initializing all the Stores at 1731505689490Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505689490Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505689490Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505689490Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505689490Cleaning up temporary data from old regions at 1731505689507 (+17 ms)Region opened successfully at 1731505689515 (+8 ms) 2024-11-13T13:48:09,516 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T13:48:09,516 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T13:48:09,516 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T13:48:09,516 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T13:48:09,516 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T13:48:09,516 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T13:48:09,516 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731505689516Disabling compacts and flushes for region at 1731505689516Disabling writes for close at 1731505689516Writing region close event to WAL at 1731505689516Closed at 1731505689516 2024-11-13T13:48:09,517 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T13:48:09,517 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T13:48:09,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T13:48:09,519 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T13:48:09,519 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T13:48:09,543 INFO [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(746): ClusterId : a8ec8e68-2e4c-43c8-95f8-f934aa5c759d 2024-11-13T13:48:09,543 INFO [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(746): ClusterId : a8ec8e68-2e4c-43c8-95f8-f934aa5c759d 2024-11-13T13:48:09,543 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T13:48:09,543 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T13:48:09,546 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T13:48:09,546 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T13:48:09,546 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T13:48:09,546 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T13:48:09,550 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T13:48:09,550 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T13:48:09,550 DEBUG [RS:1;0b62828576b1:34919 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c9a0d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b62828576b1/172.17.0.3:0 2024-11-13T13:48:09,551 DEBUG [RS:0;0b62828576b1:44471 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2e06c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b62828576b1/172.17.0.3:0 2024-11-13T13:48:09,564 DEBUG [RS:1;0b62828576b1:34919 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0b62828576b1:34919 2024-11-13T13:48:09,564 INFO [RS:1;0b62828576b1:34919 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T13:48:09,564 INFO [RS:1;0b62828576b1:34919 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T13:48:09,564 DEBUG [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T13:48:09,565 INFO [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b62828576b1,34663,1731505689137 with port=34919, startcode=1731505689218 2024-11-13T13:48:09,565 DEBUG [RS:1;0b62828576b1:34919 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T13:48:09,567 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42683, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T13:48:09,568 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b62828576b1,34919,1731505689218 2024-11-13T13:48:09,568 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] master.ServerManager(517): Registering regionserver=0b62828576b1,34919,1731505689218 2024-11-13T13:48:09,569 DEBUG [RS:0;0b62828576b1:44471 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0b62828576b1:44471 2024-11-13T13:48:09,570 INFO [RS:0;0b62828576b1:44471 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T13:48:09,570 INFO [RS:0;0b62828576b1:44471 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T13:48:09,570 DEBUG [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T13:48:09,570 DEBUG [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2 2024-11-13T13:48:09,570 DEBUG [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T13:48:09,570 DEBUG [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T13:48:09,571 INFO [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b62828576b1,34663,1731505689137 with port=44471, startcode=1731505689185 2024-11-13T13:48:09,571 DEBUG [RS:0;0b62828576b1:44471 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T13:48:09,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T13:48:09,573 DEBUG [RS:1;0b62828576b1:34919 {}] zookeeper.ZKUtil(111): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b62828576b1,34919,1731505689218 2024-11-13T13:48:09,573 WARN [RS:1;0b62828576b1:34919 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T13:48:09,573 INFO [RS:1;0b62828576b1:34919 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T13:48:09,573 DEBUG [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,34919,1731505689218 2024-11-13T13:48:09,573 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b62828576b1,34919,1731505689218] 2024-11-13T13:48:09,573 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53885, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T13:48:09,574 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b62828576b1,44471,1731505689185 2024-11-13T13:48:09,574 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] master.ServerManager(517): Registering regionserver=0b62828576b1,44471,1731505689185 2024-11-13T13:48:09,577 DEBUG [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2 2024-11-13T13:48:09,577 DEBUG [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T13:48:09,577 DEBUG [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T13:48:09,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T13:48:09,583 DEBUG [RS:0;0b62828576b1:44471 {}] zookeeper.ZKUtil(111): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b62828576b1,44471,1731505689185 2024-11-13T13:48:09,583 WARN [RS:0;0b62828576b1:44471 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T13:48:09,583 INFO [RS:0;0b62828576b1:44471 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T13:48:09,583 DEBUG [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,44471,1731505689185 2024-11-13T13:48:09,583 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b62828576b1,44471,1731505689185] 2024-11-13T13:48:09,583 INFO [RS:1;0b62828576b1:34919 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T13:48:09,585 INFO [RS:1;0b62828576b1:34919 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T13:48:09,586 INFO [RS:1;0b62828576b1:34919 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T13:48:09,586 INFO [RS:1;0b62828576b1:34919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,586 INFO [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T13:48:09,587 INFO [RS:1;0b62828576b1:34919 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T13:48:09,587 INFO [RS:1;0b62828576b1:34919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,587 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,587 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,587 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,587 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,587 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,587 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b62828576b1:0, corePoolSize=2, maxPoolSize=2 2024-11-13T13:48:09,588 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,588 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,588 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,588 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,588 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,588 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,588 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b62828576b1:0, corePoolSize=3, maxPoolSize=3 2024-11-13T13:48:09,588 DEBUG [RS:1;0b62828576b1:34919 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b62828576b1:0, corePoolSize=3, maxPoolSize=3 2024-11-13T13:48:09,591 INFO [RS:1;0b62828576b1:34919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,591 INFO [RS:1;0b62828576b1:34919 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,591 INFO [RS:1;0b62828576b1:34919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,591 INFO [RS:1;0b62828576b1:34919 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,591 INFO [RS:1;0b62828576b1:34919 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,591 INFO [RS:1;0b62828576b1:34919 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,34919,1731505689218-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T13:48:09,593 INFO [RS:0;0b62828576b1:44471 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T13:48:09,596 INFO [RS:0;0b62828576b1:44471 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T13:48:09,597 INFO [RS:0;0b62828576b1:44471 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T13:48:09,597 INFO [RS:0;0b62828576b1:44471 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,598 INFO [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T13:48:09,599 INFO [RS:0;0b62828576b1:44471 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T13:48:09,599 INFO [RS:0;0b62828576b1:44471 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,599 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,599 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,599 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,599 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,599 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,599 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b62828576b1:0, corePoolSize=2, maxPoolSize=2 2024-11-13T13:48:09,599 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,600 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,600 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,600 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,600 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,600 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:09,600 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b62828576b1:0, corePoolSize=3, maxPoolSize=3 2024-11-13T13:48:09,600 DEBUG [RS:0;0b62828576b1:44471 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b62828576b1:0, corePoolSize=3, maxPoolSize=3 2024-11-13T13:48:09,604 INFO [RS:0;0b62828576b1:44471 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,604 INFO [RS:0;0b62828576b1:44471 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,604 INFO [RS:0;0b62828576b1:44471 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,604 INFO [RS:0;0b62828576b1:44471 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,604 INFO [RS:0;0b62828576b1:44471 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,604 INFO [RS:0;0b62828576b1:44471 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,44471,1731505689185-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T13:48:09,609 INFO [RS:1;0b62828576b1:34919 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T13:48:09,609 INFO [RS:1;0b62828576b1:34919 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,34919,1731505689218-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,610 INFO [RS:1;0b62828576b1:34919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,610 INFO [RS:1;0b62828576b1:34919 {}] regionserver.Replication(171): 0b62828576b1,34919,1731505689218 started 2024-11-13T13:48:09,620 INFO [RS:0;0b62828576b1:44471 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T13:48:09,620 INFO [RS:0;0b62828576b1:44471 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,44471,1731505689185-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,621 INFO [RS:0;0b62828576b1:44471 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,621 INFO [RS:0;0b62828576b1:44471 {}] regionserver.Replication(171): 0b62828576b1,44471,1731505689185 started 2024-11-13T13:48:09,626 INFO [RS:1;0b62828576b1:34919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,626 INFO [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(1482): Serving as 0b62828576b1,34919,1731505689218, RpcServer on 0b62828576b1/172.17.0.3:34919, sessionid=0x10039661cbf0002 2024-11-13T13:48:09,627 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T13:48:09,627 DEBUG [RS:1;0b62828576b1:34919 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b62828576b1,34919,1731505689218 2024-11-13T13:48:09,627 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b62828576b1,34919,1731505689218' 2024-11-13T13:48:09,627 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T13:48:09,628 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T13:48:09,628 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T13:48:09,628 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T13:48:09,628 DEBUG [RS:1;0b62828576b1:34919 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b62828576b1,34919,1731505689218 2024-11-13T13:48:09,629 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b62828576b1,34919,1731505689218' 2024-11-13T13:48:09,629 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T13:48:09,629 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T13:48:09,630 DEBUG [RS:1;0b62828576b1:34919 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T13:48:09,630 INFO [RS:1;0b62828576b1:34919 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T13:48:09,630 INFO [RS:1;0b62828576b1:34919 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T13:48:09,638 INFO [RS:0;0b62828576b1:44471 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:09,638 INFO [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(1482): Serving as 0b62828576b1,44471,1731505689185, RpcServer on 0b62828576b1/172.17.0.3:44471, sessionid=0x10039661cbf0001 2024-11-13T13:48:09,638 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T13:48:09,639 DEBUG [RS:0;0b62828576b1:44471 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b62828576b1,44471,1731505689185 2024-11-13T13:48:09,639 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b62828576b1,44471,1731505689185' 2024-11-13T13:48:09,639 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T13:48:09,639 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T13:48:09,640 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T13:48:09,640 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T13:48:09,640 DEBUG [RS:0;0b62828576b1:44471 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b62828576b1,44471,1731505689185 2024-11-13T13:48:09,640 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b62828576b1,44471,1731505689185' 2024-11-13T13:48:09,640 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T13:48:09,640 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T13:48:09,641 DEBUG [RS:0;0b62828576b1:44471 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T13:48:09,641 INFO [RS:0;0b62828576b1:44471 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T13:48:09,641 INFO [RS:0;0b62828576b1:44471 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T13:48:09,670 WARN [0b62828576b1:34663 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T13:48:09,730 INFO [RS:1;0b62828576b1:34919 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T13:48:09,731 INFO [RS:1;0b62828576b1:34919 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0b62828576b1%2C34919%2C1731505689218, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,34919,1731505689218, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs, maxLogs=55 2024-11-13T13:48:09,742 INFO [RS:0;0b62828576b1:44471 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T13:48:09,742 INFO [RS:0;0b62828576b1:44471 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0b62828576b1%2C44471%2C1731505689185, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,44471,1731505689185, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs, maxLogs=55 2024-11-13T13:48:09,750 INFO [RS:1;0b62828576b1:34919 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,34919,1731505689218/0b62828576b1%2C34919%2C1731505689218.1731505689733 2024-11-13T13:48:09,751 DEBUG [RS:1;0b62828576b1:34919 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T13:48:09,763 INFO [RS:0;0b62828576b1:44471 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,44471,1731505689185/0b62828576b1%2C44471%2C1731505689185.1731505689744 2024-11-13T13:48:09,763 DEBUG [RS:0;0b62828576b1:44471 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T13:48:09,920 DEBUG [0b62828576b1:34663 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T13:48:09,921 DEBUG [0b62828576b1:34663 {}] balancer.BalancerClusterState(204): Hosts are {0b62828576b1=0} racks are {/default-rack=0} 2024-11-13T13:48:09,921 DEBUG [0b62828576b1:34663 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T13:48:09,921 DEBUG [0b62828576b1:34663 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T13:48:09,921 DEBUG [0b62828576b1:34663 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T13:48:09,921 DEBUG [0b62828576b1:34663 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T13:48:09,921 INFO [0b62828576b1:34663 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T13:48:09,921 INFO [0b62828576b1:34663 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T13:48:09,921 DEBUG [0b62828576b1:34663 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T13:48:09,922 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b62828576b1,34919,1731505689218 2024-11-13T13:48:09,923 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b62828576b1,34919,1731505689218, state=OPENING 2024-11-13T13:48:09,925 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T13:48:09,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:09,927 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:09,927 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:09,927 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T13:48:09,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:09,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b62828576b1,34919,1731505689218}] 2024-11-13T13:48:10,081 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T13:48:10,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50185, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T13:48:10,087 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T13:48:10,087 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T13:48:10,088 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T13:48:10,089 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0b62828576b1%2C34919%2C1731505689218.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,34919,1731505689218, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs, maxLogs=55 2024-11-13T13:48:10,117 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,34919,1731505689218/0b62828576b1%2C34919%2C1731505689218.meta.1731505690090.meta 2024-11-13T13:48:10,118 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T13:48:10,118 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T13:48:10,118 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T13:48:10,118 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T13:48:10,118 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T13:48:10,119 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T13:48:10,119 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:10,119 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T13:48:10,119 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T13:48:10,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T13:48:10,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T13:48:10,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:10,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:10,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T13:48:10,122 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T13:48:10,122 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:10,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:10,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T13:48:10,123 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T13:48:10,123 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:10,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:10,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T13:48:10,123 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T13:48:10,124 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:10,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:10,124 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T13:48:10,125 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740 2024-11-13T13:48:10,125 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740 2024-11-13T13:48:10,125 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T13:48:10,125 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T13:48:10,126 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T13:48:10,126 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T13:48:10,127 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73501930, jitterRate=0.09526410698890686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T13:48:10,127 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T13:48:10,128 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731505690119Writing region info on filesystem at 1731505690119Initializing all the Stores at 1731505690119Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505690119Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505690120 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505690120Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505690120Cleaning up temporary data from old regions at 1731505690125 (+5 ms)Running coprocessor post-open hooks at 1731505690127 (+2 ms)Region opened successfully at 1731505690128 (+1 ms) 2024-11-13T13:48:10,128 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731505690081 2024-11-13T13:48:10,132 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T13:48:10,132 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T13:48:10,132 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0b62828576b1,34919,1731505689218 2024-11-13T13:48:10,133 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b62828576b1,34919,1731505689218, state=OPEN 2024-11-13T13:48:10,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T13:48:10,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T13:48:10,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T13:48:10,136 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:10,136 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:10,136 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:10,136 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b62828576b1,34919,1731505689218 2024-11-13T13:48:10,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T13:48:10,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b62828576b1,34919,1731505689218 in 209 msec 2024-11-13T13:48:10,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T13:48:10,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 620 msec 2024-11-13T13:48:10,141 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T13:48:10,141 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T13:48:10,144 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T13:48:10,144 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b62828576b1,34919,1731505689218, seqNum=-1] 2024-11-13T13:48:10,145 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T13:48:10,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49169, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T13:48:10,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 696 msec 2024-11-13T13:48:10,153 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731505690152, completionTime=-1 2024-11-13T13:48:10,153 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-13T13:48:10,153 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T13:48:10,155 INFO [master/0b62828576b1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T13:48:10,155 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731505750155 2024-11-13T13:48:10,156 INFO [master/0b62828576b1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731505810156 2024-11-13T13:48:10,156 INFO [master/0b62828576b1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-13T13:48:10,156 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,34663,1731505689137-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:10,156 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,34663,1731505689137-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:10,156 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,34663,1731505689137-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:10,156 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0b62828576b1:34663, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:10,156 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:10,157 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:10,160 DEBUG [master/0b62828576b1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T13:48:10,163 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.896sec 2024-11-13T13:48:10,163 INFO [master/0b62828576b1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T13:48:10,163 INFO [master/0b62828576b1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T13:48:10,163 INFO [master/0b62828576b1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T13:48:10,163 INFO [master/0b62828576b1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T13:48:10,163 INFO [master/0b62828576b1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T13:48:10,163 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,34663,1731505689137-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T13:48:10,163 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,34663,1731505689137-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T13:48:10,164 DEBUG [master/0b62828576b1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T13:48:10,164 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T13:48:10,165 INFO [master/0b62828576b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,34663,1731505689137-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:10,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2129cce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T13:48:10,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b62828576b1,34663,-1 for getting cluster id 2024-11-13T13:48:10,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T13:48:10,246 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ec8e68-2e4c-43c8-95f8-f934aa5c759d' 2024-11-13T13:48:10,247 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T13:48:10,247 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ec8e68-2e4c-43c8-95f8-f934aa5c759d" 2024-11-13T13:48:10,248 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3e9287, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T13:48:10,248 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b62828576b1,34663,-1] 2024-11-13T13:48:10,248 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T13:48:10,248 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:10,250 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T13:48:10,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T13:48:10,252 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T13:48:10,254 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b62828576b1,34919,1731505689218, seqNum=-1] 2024-11-13T13:48:10,255 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T13:48:10,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T13:48:10,260 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0b62828576b1,34663,1731505689137 2024-11-13T13:48:10,260 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T13:48:10,262 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b62828576b1,34663,1731505689137 2024-11-13T13:48:10,262 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3aba8e80 2024-11-13T13:48:10,262 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T13:48:10,265 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T13:48:10,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testBlockEvictionOnGracefulStop1731505690260', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T13:48:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731505690260 2024-11-13T13:48:10,269 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731505690260 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T13:48:10,269 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:10,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731505690260" procId is: 4 2024-11-13T13:48:10,270 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731505690260 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T13:48:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T13:48:10,286 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731505690260-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 30c9fe9b60ef93f8316edfb4e8f8b5cd, NAME => 'testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731505690260', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2 2024-11-13T13:48:10,298 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731505690260-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:10,298 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731505690260-pool-0 {}] regionserver.HRegion(1722): Closing 30c9fe9b60ef93f8316edfb4e8f8b5cd, disabling compactions & flushes 2024-11-13T13:48:10,298 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731505690260-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:10,298 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731505690260-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:10,298 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731505690260-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. after waiting 0 ms 2024-11-13T13:48:10,298 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731505690260-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:10,298 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731505690260-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:10,298 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731505690260-pool-0 {}] regionserver.HRegion(1676): Region close journal for 30c9fe9b60ef93f8316edfb4e8f8b5cd: Waiting for close lock at 1731505690298Disabling compacts and flushes for region at 1731505690298Disabling writes for close at 1731505690298Writing region close event to WAL at 1731505690298Closed at 1731505690298 2024-11-13T13:48:10,300 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731505690260 execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T13:48:10,300 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731505690300"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731505690300"}]},"ts":"1731505690300"} 2024-11-13T13:48:10,303 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T13:48:10,305 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731505690260 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T13:48:10,305 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731505690260","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731505690305"}]},"ts":"1731505690305"} 2024-11-13T13:48:10,308 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731505690260, state=ENABLING in hbase:meta 2024-11-13T13:48:10,308 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0b62828576b1=0} racks are {/default-rack=0} 2024-11-13T13:48:10,309 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T13:48:10,309 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T13:48:10,309 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T13:48:10,309 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T13:48:10,309 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T13:48:10,309 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T13:48:10,309 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T13:48:10,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731505690260, region=30c9fe9b60ef93f8316edfb4e8f8b5cd, ASSIGN}] 2024-11-13T13:48:10,310 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731505690260, region=30c9fe9b60ef93f8316edfb4e8f8b5cd, ASSIGN 2024-11-13T13:48:10,311 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731505690260, region=30c9fe9b60ef93f8316edfb4e8f8b5cd, ASSIGN; state=OFFLINE, location=0b62828576b1,34919,1731505689218; forceNewPlan=false, retain=false 2024-11-13T13:48:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T13:48:10,462 INFO [0b62828576b1:34663 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T13:48:10,462 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=30c9fe9b60ef93f8316edfb4e8f8b5cd, regionState=OPENING, regionLocation=0b62828576b1,34919,1731505689218 2024-11-13T13:48:10,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731505690260, region=30c9fe9b60ef93f8316edfb4e8f8b5cd, ASSIGN because future has completed 2024-11-13T13:48:10,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30c9fe9b60ef93f8316edfb4e8f8b5cd, server=0b62828576b1,34919,1731505689218}] 2024-11-13T13:48:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T13:48:10,622 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:10,622 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 30c9fe9b60ef93f8316edfb4e8f8b5cd, NAME => 'testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd.', STARTKEY => '', ENDKEY => ''} 2024-11-13T13:48:10,623 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731505690260 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:10,623 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:10,623 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:10,623 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:10,624 INFO [StoreOpener-30c9fe9b60ef93f8316edfb4e8f8b5cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:10,628 INFO [StoreOpener-30c9fe9b60ef93f8316edfb4e8f8b5cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30c9fe9b60ef93f8316edfb4e8f8b5cd columnFamilyName family 2024-11-13T13:48:10,628 DEBUG [StoreOpener-30c9fe9b60ef93f8316edfb4e8f8b5cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:10,628 INFO [StoreOpener-30c9fe9b60ef93f8316edfb4e8f8b5cd-1 {}] regionserver.HStore(327): Store=30c9fe9b60ef93f8316edfb4e8f8b5cd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T13:48:10,629 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:10,629 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:10,630 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:10,630 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:10,630 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:10,630 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:10,642 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T13:48:10,642 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 30c9fe9b60ef93f8316edfb4e8f8b5cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70686894, jitterRate=0.05331680178642273}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T13:48:10,642 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:10,643 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 30c9fe9b60ef93f8316edfb4e8f8b5cd: Running coprocessor pre-open hook at 1731505690623Writing region info on filesystem at 1731505690623Initializing all the Stores at 1731505690623Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505690624 (+1 ms)Cleaning up temporary data from old regions at 1731505690630 (+6 ms)Running coprocessor post-open hooks at 1731505690642 (+12 ms)Region opened successfully at 1731505690643 (+1 ms) 2024-11-13T13:48:10,644 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd., pid=6, masterSystemTime=1731505690619 2024-11-13T13:48:10,647 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:10,647 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:10,648 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=30c9fe9b60ef93f8316edfb4e8f8b5cd, regionState=OPEN, openSeqNum=2, regionLocation=0b62828576b1,34919,1731505689218 2024-11-13T13:48:10,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30c9fe9b60ef93f8316edfb4e8f8b5cd, server=0b62828576b1,34919,1731505689218 because future has completed 2024-11-13T13:48:10,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T13:48:10,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 30c9fe9b60ef93f8316edfb4e8f8b5cd, server=0b62828576b1,34919,1731505689218 in 186 msec 2024-11-13T13:48:10,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T13:48:10,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731505690260, region=30c9fe9b60ef93f8316edfb4e8f8b5cd, ASSIGN in 344 msec 2024-11-13T13:48:10,656 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731505690260 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T13:48:10,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731505690260","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731505690656"}]},"ts":"1731505690656"} 2024-11-13T13:48:10,659 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731505690260, state=ENABLED in hbase:meta 2024-11-13T13:48:10,660 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731505690260 execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T13:48:10,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731505690260 in 394 msec 2024-11-13T13:48:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T13:48:10,902 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731505690260 completed 2024-11-13T13:48:10,902 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731505690260 get assigned. Timeout = 60000ms 2024-11-13T13:48:10,902 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T13:48:10,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731505690260 assigned to meta. Checking AM states. 2024-11-13T13:48:10,907 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T13:48:10,907 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731505690260 assigned. 2024-11-13T13:48:10,911 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731505690260', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd., hostname=0b62828576b1,34919,1731505689218, seqNum=2] 2024-11-13T13:48:10,916 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 30c9fe9b60ef93f8316edfb4e8f8b5cd 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-13T13:48:10,927 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d45353417cd5452d858c26d97ce53f3b 2024-11-13T13:48:10,941 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/.tmp/family/d45353417cd5452d858c26d97ce53f3b is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-13T13:48:10,942 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/.tmp/family/d45353417cd5452d858c26d97ce53f3b 2024-11-13T13:48:10,944 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/.tmp/family/d45353417cd5452d858c26d97ce53f3b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/family/d45353417cd5452d858c26d97ce53f3b 2024-11-13T13:48:10,946 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/family/d45353417cd5452d858c26d97ce53f3b, delay=943 ms 2024-11-13T13:48:10,947 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/family/d45353417cd5452d858c26d97ce53f3b, entries=2, sequenceid=6, filesize=4.9 K 2024-11-13T13:48:10,947 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 30c9fe9b60ef93f8316edfb4e8f8b5cd in 31ms, sequenceid=6, compaction requested=false 2024-11-13T13:48:10,947 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731505690260' 2024-11-13T13:48:10,948 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 30c9fe9b60ef93f8316edfb4e8f8b5cd: 2024-11-13T13:48:11,362 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T13:48:11,890 DEBUG [hfile-prefetch-1731505688557 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d45353417cd5452d858c26d97ce53f3b_0. 2024-11-13T13:48:11,890 DEBUG [hfile-prefetch-1731505688557 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d45353417cd5452d858c26d97ce53f3b_117. 2024-11-13T13:48:11,891 DEBUG [hfile-prefetch-1731505688557 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/family/d45353417cd5452d858c26d97ce53f3b, with total blocks 2, and data blocks 1 2024-11-13T13:48:11,891 DEBUG [hfile-prefetch-1731505688557 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-13T13:48:11,891 DEBUG [hfile-prefetch-1731505688557 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/family/d45353417cd5452d858c26d97ce53f3b, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-13T13:48:11,891 DEBUG [hfile-prefetch-1731505688557 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/family/d45353417cd5452d858c26d97ce53f3b. Will sleep 100ms and try the verification again. 2024-11-13T13:48:11,897 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-13T13:48:11,897 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-13T13:48:11,897 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T13:48:11,897 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T13:48:11,900 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T13:48:11,900 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731505691897 2024-11-13T13:48:11,900 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-13T13:48:11,948 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;0b62828576b1:34919,5,FailOnTimeoutGroup] 2024-11-13T13:48:11,949 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b62828576b1,34919,1731505689218' ***** 2024-11-13T13:48:11,949 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-13T13:48:11,949 INFO [RS:1;0b62828576b1:34919 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T13:48:11,949 INFO [RS:1;0b62828576b1:34919 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.persistence 2024-11-13T13:48:11,949 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T13:48:11,949 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T13:48:11,949 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T13:48:11,949 INFO [RS:1;0b62828576b1:34919 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T13:48:11,950 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T13:48:11,960 DEBUG [RS:1;0b62828576b1:34919 {}] bucket.BucketCache(1488): Thread RS:1;0b62828576b1:34919 started persisting bucket cache to file 2024-11-13T13:48:11,960 DEBUG [RS:1;0b62828576b1:34919 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T13:48:11,960 DEBUG [RS:1;0b62828576b1:34919 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T13:48:11,968 DEBUG [RS:1;0b62828576b1:34919 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-13T13:48:11,968 DEBUG [RS:1;0b62828576b1:34919 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731505691960 2024-11-13T13:48:11,968 DEBUG [RS:1;0b62828576b1:34919 {}] bucket.BucketCache(1510): Thread RS:1;0b62828576b1:34919 finished persisting bucket cache to file, renaming 2024-11-13T13:48:11,968 INFO [RS:1;0b62828576b1:34919 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T13:48:11,968 INFO [RS:1;0b62828576b1:34919 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T13:48:11,968 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T13:48:11,968 INFO [RS:1;0b62828576b1:34919 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T13:48:11,968 INFO [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(3091): Received CLOSE for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:11,969 INFO [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(959): stopping server 0b62828576b1,34919,1731505689218 2024-11-13T13:48:11,969 INFO [RS:1;0b62828576b1:34919 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T13:48:11,969 INFO [RS:1;0b62828576b1:34919 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0b62828576b1:34919. 2024-11-13T13:48:11,969 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T13:48:11,969 DEBUG [RS:1;0b62828576b1:34919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T13:48:11,969 DEBUG [RS:1;0b62828576b1:34919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:11,969 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 30c9fe9b60ef93f8316edfb4e8f8b5cd, disabling compactions & flushes 2024-11-13T13:48:11,969 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:11,969 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:11,969 INFO [RS:1;0b62828576b1:34919 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T13:48:11,969 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. after waiting 0 ms 2024-11-13T13:48:11,969 INFO [RS:1;0b62828576b1:34919 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T13:48:11,969 INFO [RS:1;0b62828576b1:34919 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T13:48:11,969 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:11,969 INFO [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T13:48:11,970 INFO [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T13:48:11,970 DEBUG [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 30c9fe9b60ef93f8316edfb4e8f8b5cd=testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd.} 2024-11-13T13:48:11,970 DEBUG [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:11,970 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T13:48:11,970 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T13:48:11,970 DEBUG [StoreFileCloser-30c9fe9b60ef93f8316edfb4e8f8b5cd-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/family/d45353417cd5452d858c26d97ce53f3b 2024-11-13T13:48:11,970 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T13:48:11,970 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T13:48:11,970 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T13:48:11,970 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T13:48:11,970 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-13T13:48:11,972 WARN [hfile-prefetch-1731505688557 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/family/d45353417cd5452d858c26d97ce53f3b, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-13T13:48:11,973 DEBUG [hfile-prefetch-1731505688557 {}] hfile.PrefetchExecutor(121): Prefetch completed for d45353417cd5452d858c26d97ce53f3b 2024-11-13T13:48:11,975 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-13T13:48:11,976 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:11,976 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 30c9fe9b60ef93f8316edfb4e8f8b5cd: Waiting for close lock at 1731505691969Running coprocessor pre-close hooks at 1731505691969Disabling compacts and flushes for region at 1731505691969Disabling writes for close at 1731505691969Writing region close event to WAL at 1731505691970 (+1 ms)Running coprocessor post-close hooks at 1731505691976 (+6 ms)Closed at 1731505691976 2024-11-13T13:48:11,976 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:11,982 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 72b24f9dadaa4217bbde9650730f1ae4 2024-11-13T13:48:11,994 INFO [regionserver/0b62828576b1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T13:48:11,997 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/info/72b24f9dadaa4217bbde9650730f1ae4 is 209, key is testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd./info:regioninfo/1731505690647/Put/seqid=0 2024-11-13T13:48:11,998 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/info/72b24f9dadaa4217bbde9650730f1ae4 2024-11-13T13:48:12,008 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6beaf43c521e48a591e0b1fd2a2d48a8 2024-11-13T13:48:12,023 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/ns/6beaf43c521e48a591e0b1fd2a2d48a8 is 43, key is default/ns:d/1731505690147/Put/seqid=0 2024-11-13T13:48:12,024 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/ns/6beaf43c521e48a591e0b1fd2a2d48a8 2024-11-13T13:48:12,035 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e26d8068ec2c4f998e89466d185b6da5 2024-11-13T13:48:12,048 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/table/e26d8068ec2c4f998e89466d185b6da5 is 80, key is testBlockEvictionOnGracefulStop1731505690260/table:state/1731505690656/Put/seqid=0 2024-11-13T13:48:12,049 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/table/e26d8068ec2c4f998e89466d185b6da5 2024-11-13T13:48:12,052 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/info/72b24f9dadaa4217bbde9650730f1ae4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/72b24f9dadaa4217bbde9650730f1ae4 2024-11-13T13:48:12,053 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/72b24f9dadaa4217bbde9650730f1ae4, delay=965 ms 2024-11-13T13:48:12,054 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/72b24f9dadaa4217bbde9650730f1ae4, entries=10, sequenceid=11, filesize=7.0 K 2024-11-13T13:48:12,055 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/ns/6beaf43c521e48a591e0b1fd2a2d48a8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/ns/6beaf43c521e48a591e0b1fd2a2d48a8 2024-11-13T13:48:12,056 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/ns/6beaf43c521e48a591e0b1fd2a2d48a8, delay=942 ms 2024-11-13T13:48:12,057 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/ns/6beaf43c521e48a591e0b1fd2a2d48a8, entries=2, sequenceid=11, filesize=5.0 K 2024-11-13T13:48:12,058 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/table/e26d8068ec2c4f998e89466d185b6da5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/table/e26d8068ec2c4f998e89466d185b6da5 2024-11-13T13:48:12,059 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/table/e26d8068ec2c4f998e89466d185b6da5, delay=944 ms 2024-11-13T13:48:12,060 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/table/e26d8068ec2c4f998e89466d185b6da5, entries=2, sequenceid=11, filesize=5.3 K 2024-11-13T13:48:12,060 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=11, compaction requested=false 2024-11-13T13:48:12,061 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/72b24f9dadaa4217bbde9650730f1ae4 2024-11-13T13:48:12,062 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/ns/6beaf43c521e48a591e0b1fd2a2d48a8 2024-11-13T13:48:12,063 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/table/e26d8068ec2c4f998e89466d185b6da5 2024-11-13T13:48:12,068 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-13T13:48:12,069 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T13:48:12,069 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T13:48:12,069 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731505691970Running coprocessor pre-close hooks at 1731505691970Disabling compacts and flushes for region at 1731505691970Disabling writes for close at 1731505691970Obtaining lock to block concurrent updates at 1731505691970Preparing flush snapshotting stores in 1588230740 at 1731505691970Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731505691971 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731505691971Flushing 1588230740/info: creating writer at 1731505691971Flushing 1588230740/info: appending metadata at 1731505691996 (+25 ms)Flushing 1588230740/info: closing flushed file at 1731505691996Flushing 1588230740/ns: creating writer at 1731505692001 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731505692022 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731505692022Flushing 1588230740/table: creating writer at 1731505692027 (+5 ms)Flushing 1588230740/table: appending metadata at 1731505692048 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731505692048Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19a541dd: reopening flushed file at 1731505692052 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18f5cae: reopening flushed file at 1731505692055 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bad236f: reopening flushed file at 1731505692057 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=11, compaction requested=false at 1731505692060 (+3 ms)Writing region close event to WAL at 1731505692063 (+3 ms)Running coprocessor post-close hooks at 1731505692069 (+6 ms)Closed at 1731505692069 2024-11-13T13:48:12,069 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T13:48:12,170 INFO [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(976): stopping server 0b62828576b1,34919,1731505689218; all regions closed. 2024-11-13T13:48:12,172 DEBUG [RS:1;0b62828576b1:34919 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs 2024-11-13T13:48:12,172 INFO [RS:1;0b62828576b1:34919 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b62828576b1%2C34919%2C1731505689218.meta:.meta(num 1731505690090) 2024-11-13T13:48:12,174 DEBUG [RS:1;0b62828576b1:34919 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs 2024-11-13T13:48:12,174 INFO [RS:1;0b62828576b1:34919 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b62828576b1%2C34919%2C1731505689218:(num 1731505689733) 2024-11-13T13:48:12,174 DEBUG [RS:1;0b62828576b1:34919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:12,174 INFO [RS:1;0b62828576b1:34919 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T13:48:12,175 INFO [RS:1;0b62828576b1:34919 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T13:48:12,175 INFO [RS:1;0b62828576b1:34919 {}] hbase.ChoreService(370): Chore service for: regionserver/0b62828576b1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T13:48:12,175 INFO [RS:1;0b62828576b1:34919 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T13:48:12,175 INFO [regionserver/0b62828576b1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T13:48:12,175 INFO [RS:1;0b62828576b1:34919 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34919 2024-11-13T13:48:12,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b62828576b1,34919,1731505689218 2024-11-13T13:48:12,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T13:48:12,180 INFO [RS:1;0b62828576b1:34919 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T13:48:12,181 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b62828576b1,34919,1731505689218] 2024-11-13T13:48:12,183 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b62828576b1,34919,1731505689218 already deleted, retry=false 2024-11-13T13:48:12,183 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 0b62828576b1,34919,1731505689218 on 0b62828576b1,34663,1731505689137 2024-11-13T13:48:12,188 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 0b62828576b1,34919,1731505689218, splitWal=true, meta=true 2024-11-13T13:48:12,190 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 0b62828576b1,34919,1731505689218, splitWal=true, meta=true 2024-11-13T13:48:12,191 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 0b62828576b1,34919,1731505689218 (carryingMeta=true) 0b62828576b1,34919,1731505689218/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4c568027[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T13:48:12,191 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 0b62828576b1,34919,1731505689218, splitWal=true, meta=true, isMeta: true 2024-11-13T13:48:12,191 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,34919,1731505689218-splitting 2024-11-13T13:48:12,192 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,34919,1731505689218-splitting dir is empty, no logs to split. 2024-11-13T13:48:12,192 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0b62828576b1,34919,1731505689218 WAL count=0, meta=true 2024-11-13T13:48:12,194 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,34919,1731505689218-splitting dir is empty, no logs to split. 2024-11-13T13:48:12,194 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0b62828576b1,34919,1731505689218 WAL count=0, meta=true 2024-11-13T13:48:12,194 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 0b62828576b1,34919,1731505689218 WAL splitting is done? wals=0, meta=true 2024-11-13T13:48:12,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T13:48:12,196 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T13:48:12,197 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T13:48:12,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:12,282 INFO [RS:1;0b62828576b1:34919 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T13:48:12,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34919-0x10039661cbf0002, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:12,282 INFO [RS:1;0b62828576b1:34919 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b62828576b1,34919,1731505689218; zookeeper connection closed. 2024-11-13T13:48:12,282 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7001c95b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7001c95b 2024-11-13T13:48:12,347 DEBUG [0b62828576b1:34663 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T13:48:12,348 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b62828576b1,44471,1731505689185 2024-11-13T13:48:12,349 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b62828576b1,44471,1731505689185, state=OPENING 2024-11-13T13:48:12,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T13:48:12,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T13:48:12,351 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T13:48:12,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:12,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:12,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b62828576b1,44471,1731505689185}] 2024-11-13T13:48:12,465 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b62828576b1:0 server-side Connection retries=45 2024-11-13T13:48:12,465 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:12,466 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:12,466 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T13:48:12,466 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T13:48:12,466 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T13:48:12,466 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T13:48:12,466 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T13:48:12,467 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45901 2024-11-13T13:48:12,468 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45901 connecting to ZooKeeper ensemble=127.0.0.1:58953 2024-11-13T13:48:12,468 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:12,468 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:12,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:459010x0, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T13:48:12,472 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45901-0x10039661cbf0003 connected 2024-11-13T13:48:12,472 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45901-0x10039661cbf0003, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-13T13:48:12,472 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-13T13:48:12,473 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T13:48:12,474 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.cache 2024-11-13T13:48:12,474 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T13:48:12,474 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T13:48:12,474 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T13:48:12,474 INFO [Thread-207 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T13:48:12,475 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T13:48:12,475 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T13:48:12,475 INFO [Thread-207 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-13T13:48:12,476 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45901-0x10039661cbf0003, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T13:48:12,477 INFO [Thread-207 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-13T13:48:12,478 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45901-0x10039661cbf0003, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T13:48:12,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45901 2024-11-13T13:48:12,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45901 2024-11-13T13:48:12,481 DEBUG [Thread-207 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-13T13:48:12,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45901 2024-11-13T13:48:12,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45901 2024-11-13T13:48:12,481 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T13:48:12,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45901 2024-11-13T13:48:12,481 INFO [Thread-207 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-13T13:48:12,482 INFO [Thread-207 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.persistence is complete. 2024-11-13T13:48:12,483 INFO [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(746): ClusterId : a8ec8e68-2e4c-43c8-95f8-f934aa5c759d 2024-11-13T13:48:12,483 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T13:48:12,485 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T13:48:12,485 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T13:48:12,488 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T13:48:12,488 DEBUG [RS:2;0b62828576b1:45901 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a9d8933, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b62828576b1/172.17.0.3:0 2024-11-13T13:48:12,502 DEBUG [RS:2;0b62828576b1:45901 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;0b62828576b1:45901 2024-11-13T13:48:12,502 INFO [RS:2;0b62828576b1:45901 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T13:48:12,502 INFO [RS:2;0b62828576b1:45901 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T13:48:12,502 DEBUG [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T13:48:12,503 INFO [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b62828576b1,34663,1731505689137 with port=45901, startcode=1731505692465 2024-11-13T13:48:12,503 DEBUG [RS:2;0b62828576b1:45901 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T13:48:12,504 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T13:48:12,505 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48069, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T13:48:12,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34519, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T13:48:12,506 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b62828576b1,45901,1731505692465 2024-11-13T13:48:12,506 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] master.ServerManager(517): Registering regionserver=0b62828576b1,45901,1731505692465 2024-11-13T13:48:12,508 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T13:48:12,508 DEBUG [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2 2024-11-13T13:48:12,508 DEBUG [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T13:48:12,508 DEBUG [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T13:48:12,508 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T13:48:12,508 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T13:48:12,509 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0b62828576b1%2C44471%2C1731505689185.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,44471,1731505689185, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs, maxLogs=55 2024-11-13T13:48:12,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T13:48:12,511 DEBUG [RS:2;0b62828576b1:45901 {}] zookeeper.ZKUtil(111): regionserver:45901-0x10039661cbf0003, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b62828576b1,45901,1731505692465 2024-11-13T13:48:12,511 WARN [RS:2;0b62828576b1:45901 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T13:48:12,511 INFO [RS:2;0b62828576b1:45901 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T13:48:12,511 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b62828576b1,45901,1731505692465] 2024-11-13T13:48:12,511 DEBUG [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,45901,1731505692465 2024-11-13T13:48:12,516 INFO [RS:2;0b62828576b1:45901 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T13:48:12,518 INFO [RS:2;0b62828576b1:45901 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T13:48:12,519 INFO [RS:2;0b62828576b1:45901 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T13:48:12,519 INFO [RS:2;0b62828576b1:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:12,519 INFO [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T13:48:12,519 INFO [RS:2;0b62828576b1:45901 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T13:48:12,520 INFO [RS:2;0b62828576b1:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b62828576b1:0, corePoolSize=2, maxPoolSize=2 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b62828576b1:0, corePoolSize=1, maxPoolSize=1 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b62828576b1:0, corePoolSize=3, maxPoolSize=3 2024-11-13T13:48:12,520 DEBUG [RS:2;0b62828576b1:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b62828576b1:0, corePoolSize=3, maxPoolSize=3 2024-11-13T13:48:12,523 INFO [RS:2;0b62828576b1:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:12,523 INFO [RS:2;0b62828576b1:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:12,523 INFO [RS:2;0b62828576b1:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:12,523 INFO [RS:2;0b62828576b1:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:12,523 INFO [RS:2;0b62828576b1:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:12,524 INFO [RS:2;0b62828576b1:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,45901,1731505692465-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T13:48:12,537 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,44471,1731505689185/0b62828576b1%2C44471%2C1731505689185.meta.1731505692511.meta 2024-11-13T13:48:12,538 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T13:48:12,538 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T13:48:12,538 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T13:48:12,538 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T13:48:12,538 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T13:48:12,538 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T13:48:12,539 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:12,539 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T13:48:12,539 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T13:48:12,539 INFO [RS:2;0b62828576b1:45901 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T13:48:12,540 INFO [RS:2;0b62828576b1:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b62828576b1,45901,1731505692465-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:12,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T13:48:12,540 INFO [RS:2;0b62828576b1:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:12,540 INFO [RS:2;0b62828576b1:45901 {}] regionserver.Replication(171): 0b62828576b1,45901,1731505692465 started 2024-11-13T13:48:12,541 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T13:48:12,541 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:12,543 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/72b24f9dadaa4217bbde9650730f1ae4, delay=987 ms 2024-11-13T13:48:12,544 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/72b24f9dadaa4217bbde9650730f1ae4 2024-11-13T13:48:12,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:12,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T13:48:12,546 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T13:48:12,546 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:12,548 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/ns/6beaf43c521e48a591e0b1fd2a2d48a8, delay=986 ms 2024-11-13T13:48:12,549 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/ns/6beaf43c521e48a591e0b1fd2a2d48a8 2024-11-13T13:48:12,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:12,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T13:48:12,551 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T13:48:12,551 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:12,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:12,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T13:48:12,552 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T13:48:12,552 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:12,555 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/table/e26d8068ec2c4f998e89466d185b6da5, delay=929 ms 2024-11-13T13:48:12,557 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/table/e26d8068ec2c4f998e89466d185b6da5 2024-11-13T13:48:12,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T13:48:12,557 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T13:48:12,558 INFO [RS:2;0b62828576b1:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T13:48:12,558 INFO [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(1482): Serving as 0b62828576b1,45901,1731505692465, RpcServer on 0b62828576b1/172.17.0.3:45901, sessionid=0x10039661cbf0003 2024-11-13T13:48:12,558 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T13:48:12,558 DEBUG [RS:2;0b62828576b1:45901 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b62828576b1,45901,1731505692465 2024-11-13T13:48:12,558 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b62828576b1,45901,1731505692465' 2024-11-13T13:48:12,558 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T13:48:12,558 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740 2024-11-13T13:48:12,559 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740 2024-11-13T13:48:12,559 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T13:48:12,559 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T13:48:12,559 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T13:48:12,559 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T13:48:12,559 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T13:48:12,559 DEBUG [RS:2;0b62828576b1:45901 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b62828576b1,45901,1731505692465 2024-11-13T13:48:12,559 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b62828576b1,45901,1731505692465' 2024-11-13T13:48:12,560 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T13:48:12,560 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T13:48:12,560 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T13:48:12,561 DEBUG [RS:2;0b62828576b1:45901 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T13:48:12,561 INFO [RS:2;0b62828576b1:45901 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T13:48:12,561 INFO [RS:2;0b62828576b1:45901 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T13:48:12,561 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T13:48:12,562 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65957044, jitterRate=-0.017163455486297607}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T13:48:12,562 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T13:48:12,563 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731505692539Writing region info on filesystem at 1731505692539Initializing all the Stores at 1731505692539Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505692539Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505692539Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505692539Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731505692539Cleaning up temporary data from old regions at 1731505692559 (+20 ms)Running coprocessor post-open hooks at 1731505692562 (+3 ms)Region opened successfully at 1731505692563 (+1 ms) 2024-11-13T13:48:12,564 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731505692503 2024-11-13T13:48:12,568 DEBUG [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T13:48:12,568 INFO [RS_OPEN_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T13:48:12,568 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=0b62828576b1,44471,1731505689185 2024-11-13T13:48:12,569 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b62828576b1,44471,1731505689185, state=OPEN 2024-11-13T13:48:12,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T13:48:12,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T13:48:12,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x10039661cbf0003, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T13:48:12,571 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b62828576b1,44471,1731505689185 2024-11-13T13:48:12,571 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:12,571 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:12,571 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T13:48:12,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-13T13:48:12,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b62828576b1,44471,1731505689185 in 220 msec 2024-11-13T13:48:12,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T13:48:12,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 379 msec 2024-11-13T13:48:12,576 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 0b62828576b1,34919,1731505689218 had 2 regions 2024-11-13T13:48:12,577 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 0b62828576b1,34919,1731505689218, splitWal=true, meta=true, isMeta: false 2024-11-13T13:48:12,578 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,34919,1731505689218-splitting dir is empty, no logs to split. 2024-11-13T13:48:12,578 INFO [PEWorker-3 {}] master.SplitWALManager(105): 0b62828576b1,34919,1731505689218 WAL count=0, meta=false 2024-11-13T13:48:12,579 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,34919,1731505689218-splitting dir is empty, no logs to split. 2024-11-13T13:48:12,579 INFO [PEWorker-3 {}] master.SplitWALManager(105): 0b62828576b1,34919,1731505689218 WAL count=0, meta=false 2024-11-13T13:48:12,579 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 0b62828576b1,34919,1731505689218 WAL splitting is done? wals=0, meta=false 2024-11-13T13:48:12,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731505690260, region=30c9fe9b60ef93f8316edfb4e8f8b5cd, ASSIGN}] 2024-11-13T13:48:12,581 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731505690260, region=30c9fe9b60ef93f8316edfb4e8f8b5cd, ASSIGN 2024-11-13T13:48:12,582 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731505690260, region=30c9fe9b60ef93f8316edfb4e8f8b5cd, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T13:48:12,662 INFO [RS:2;0b62828576b1:45901 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T13:48:12,662 INFO [RS:2;0b62828576b1:45901 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0b62828576b1%2C45901%2C1731505692465, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,45901,1731505692465, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs, maxLogs=55 2024-11-13T13:48:12,689 INFO [RS:2;0b62828576b1:45901 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/WALs/0b62828576b1,45901,1731505692465/0b62828576b1%2C45901%2C1731505692465.1731505692664 2024-11-13T13:48:12,690 DEBUG [RS:2;0b62828576b1:45901 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T13:48:12,732 DEBUG [0b62828576b1:34663 {}] balancer.BalancerClusterState(204): Hosts are {0b62828576b1=0} racks are {/default-rack=0} 2024-11-13T13:48:12,733 DEBUG [0b62828576b1:34663 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T13:48:12,733 DEBUG [0b62828576b1:34663 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T13:48:12,733 DEBUG [0b62828576b1:34663 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T13:48:12,733 DEBUG [0b62828576b1:34663 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T13:48:12,733 INFO [0b62828576b1:34663 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T13:48:12,733 INFO [0b62828576b1:34663 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T13:48:12,733 DEBUG [0b62828576b1:34663 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T13:48:12,733 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=30c9fe9b60ef93f8316edfb4e8f8b5cd, regionState=OPENING, regionLocation=0b62828576b1,45901,1731505692465 2024-11-13T13:48:12,735 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0b62828576b1:34919 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0b62828576b1/172.17.0.3:34919 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T13:48:12,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0b62828576b1,34919,1731505689218, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0b62828576b1,34919,1731505689218, seqNum=-1, error=java.net.ConnectException: Call to address=0b62828576b1:34919 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0b62828576b1/172.17.0.3:34919 2024-11-13T13:48:12,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0b62828576b1,34919,1731505689218, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T13:48:12,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0b62828576b1,34919,1731505689218, seqNum=-1 from cache 2024-11-13T13:48:12,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 0b62828576b1:34919 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0b62828576b1/172.17.0.3:34919 2024-11-13T13:48:12,841 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T13:48:12,841 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b62828576b1,44471,1731505689185, seqNum=-1] 2024-11-13T13:48:12,841 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T13:48:12,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55895, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T13:48:12,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731505690260, region=30c9fe9b60ef93f8316edfb4e8f8b5cd, ASSIGN because future has completed 2024-11-13T13:48:12,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30c9fe9b60ef93f8316edfb4e8f8b5cd, server=0b62828576b1,45901,1731505692465}] 2024-11-13T13:48:12,999 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T13:48:13,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55841, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T13:48:13,003 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:13,003 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 30c9fe9b60ef93f8316edfb4e8f8b5cd, NAME => 'testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd.', STARTKEY => '', ENDKEY => ''} 2024-11-13T13:48:13,004 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731505690260 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,004 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T13:48:13,004 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,004 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,005 INFO [StoreOpener-30c9fe9b60ef93f8316edfb4e8f8b5cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,006 INFO [StoreOpener-30c9fe9b60ef93f8316edfb4e8f8b5cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30c9fe9b60ef93f8316edfb4e8f8b5cd columnFamilyName family 2024-11-13T13:48:13,006 DEBUG [StoreOpener-30c9fe9b60ef93f8316edfb4e8f8b5cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T13:48:13,008 DEBUG [StoreFileOpener-30c9fe9b60ef93f8316edfb4e8f8b5cd-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/family/d45353417cd5452d858c26d97ce53f3b, delay=983 ms 2024-11-13T13:48:13,009 DEBUG [StoreOpener-30c9fe9b60ef93f8316edfb4e8f8b5cd-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/family/d45353417cd5452d858c26d97ce53f3b 2024-11-13T13:48:13,009 INFO [StoreOpener-30c9fe9b60ef93f8316edfb4e8f8b5cd-1 {}] regionserver.HStore(327): Store=30c9fe9b60ef93f8316edfb4e8f8b5cd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T13:48:13,010 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,010 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,011 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,011 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,011 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,012 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,012 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 30c9fe9b60ef93f8316edfb4e8f8b5cd; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66892794, jitterRate=-0.003219693899154663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T13:48:13,012 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,013 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 30c9fe9b60ef93f8316edfb4e8f8b5cd: Running coprocessor pre-open hook at 1731505693004Writing region info on filesystem at 1731505693004Initializing all the Stores at 1731505693005 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731505693005Cleaning up temporary data from old regions at 1731505693011 (+6 ms)Running coprocessor post-open hooks at 1731505693013 (+2 ms)Region opened successfully at 1731505693013 2024-11-13T13:48:13,014 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd., pid=11, masterSystemTime=1731505692998 2024-11-13T13:48:13,017 DEBUG [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:13,017 INFO [RS_OPEN_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:13,017 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=30c9fe9b60ef93f8316edfb4e8f8b5cd, regionState=OPEN, openSeqNum=10, regionLocation=0b62828576b1,45901,1731505692465 2024-11-13T13:48:13,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30c9fe9b60ef93f8316edfb4e8f8b5cd, server=0b62828576b1,45901,1731505692465 because future has completed 2024-11-13T13:48:13,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T13:48:13,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 30c9fe9b60ef93f8316edfb4e8f8b5cd, server=0b62828576b1,45901,1731505692465 in 174 msec 2024-11-13T13:48:13,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-13T13:48:13,025 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 0b62828576b1,34919,1731505689218 after splitting done 2024-11-13T13:48:13,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731505690260, region=30c9fe9b60ef93f8316edfb4e8f8b5cd, ASSIGN in 442 msec 2024-11-13T13:48:13,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 0b62828576b1,34919,1731505689218, splitWal=true, meta=true in 841 msec 2024-11-13T13:48:13,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T13:48:13,058 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T13:48:13,058 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T13:48:13,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:13,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:13,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T13:48:13,059 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T13:48:13,059 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2023292426, stopped=false 2024-11-13T13:48:13,059 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0b62828576b1,34663,1731505689137 2024-11-13T13:48:13,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T13:48:13,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x10039661cbf0003, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T13:48:13,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T13:48:13,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x10039661cbf0003, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:13,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:13,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:13,061 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T13:48:13,062 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T13:48:13,062 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T13:48:13,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:13,062 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T13:48:13,062 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b62828576b1,44471,1731505689185' ***** 2024-11-13T13:48:13,062 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T13:48:13,063 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T13:48:13,063 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b62828576b1,45901,1731505692465' ***** 2024-11-13T13:48:13,063 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T13:48:13,063 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45901-0x10039661cbf0003, quorum=127.0.0.1:58953, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T13:48:13,063 INFO [RS:2;0b62828576b1:45901 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T13:48:13,063 INFO [RS:2;0b62828576b1:45901 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.persistence 2024-11-13T13:48:13,063 INFO [RS:0;0b62828576b1:44471 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T13:48:13,063 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T13:48:13,063 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T13:48:13,063 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T13:48:13,063 INFO [RS:0;0b62828576b1:44471 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/627764bd-72cd-55ed-0777-7d327eec7752/bucket.persistence 2024-11-13T13:48:13,063 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T13:48:13,063 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T13:48:13,064 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T13:48:13,064 INFO [RS:2;0b62828576b1:45901 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T13:48:13,064 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T13:48:13,065 INFO [RS:0;0b62828576b1:44471 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T13:48:13,065 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T13:48:13,074 DEBUG [RS:2;0b62828576b1:45901 {}] bucket.BucketCache(1488): Thread RS:2;0b62828576b1:45901 started persisting bucket cache to file 2024-11-13T13:48:13,074 DEBUG [RS:2;0b62828576b1:45901 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T13:48:13,074 DEBUG [RS:2;0b62828576b1:45901 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T13:48:13,075 DEBUG [RS:0;0b62828576b1:44471 {}] bucket.BucketCache(1488): Thread RS:0;0b62828576b1:44471 started persisting bucket cache to file 2024-11-13T13:48:13,075 DEBUG [RS:0;0b62828576b1:44471 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T13:48:13,075 DEBUG [RS:0;0b62828576b1:44471 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T13:48:13,078 DEBUG [RS:2;0b62828576b1:45901 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-13T13:48:13,078 DEBUG [RS:2;0b62828576b1:45901 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731505693074 2024-11-13T13:48:13,078 DEBUG [RS:2;0b62828576b1:45901 {}] bucket.BucketCache(1510): Thread RS:2;0b62828576b1:45901 finished persisting bucket cache to file, renaming 2024-11-13T13:48:13,078 INFO [RS:2;0b62828576b1:45901 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T13:48:13,078 INFO [RS:2;0b62828576b1:45901 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T13:48:13,078 INFO [RS:2;0b62828576b1:45901 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T13:48:13,078 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T13:48:13,078 INFO [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(3091): Received CLOSE for 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,078 DEBUG [RS:0;0b62828576b1:44471 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T13:48:13,079 DEBUG [RS:0;0b62828576b1:44471 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731505693075 2024-11-13T13:48:13,079 DEBUG [RS:0;0b62828576b1:44471 {}] bucket.BucketCache(1510): Thread RS:0;0b62828576b1:44471 finished persisting bucket cache to file, renaming 2024-11-13T13:48:13,079 INFO [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(959): stopping server 0b62828576b1,45901,1731505692465 2024-11-13T13:48:13,079 INFO [RS:2;0b62828576b1:45901 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T13:48:13,079 INFO [RS:0;0b62828576b1:44471 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T13:48:13,079 INFO [RS:2;0b62828576b1:45901 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;0b62828576b1:45901. 2024-11-13T13:48:13,079 INFO [RS:0;0b62828576b1:44471 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T13:48:13,079 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T13:48:13,079 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T13:48:13,079 DEBUG [RS:2;0b62828576b1:45901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T13:48:13,079 INFO [RS:0;0b62828576b1:44471 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T13:48:13,079 INFO [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(959): stopping server 0b62828576b1,44471,1731505689185 2024-11-13T13:48:13,079 DEBUG [RS:2;0b62828576b1:45901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:13,079 INFO [RS:0;0b62828576b1:44471 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T13:48:13,079 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 30c9fe9b60ef93f8316edfb4e8f8b5cd, disabling compactions & flushes 2024-11-13T13:48:13,079 INFO [RS:0;0b62828576b1:44471 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0b62828576b1:44471. 2024-11-13T13:48:13,079 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:13,079 DEBUG [RS:0;0b62828576b1:44471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T13:48:13,079 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:13,079 INFO [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T13:48:13,079 DEBUG [RS:0;0b62828576b1:44471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:13,079 DEBUG [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(1325): Online Regions={30c9fe9b60ef93f8316edfb4e8f8b5cd=testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd.} 2024-11-13T13:48:13,079 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. after waiting 0 ms 2024-11-13T13:48:13,080 DEBUG [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(1351): Waiting on 30c9fe9b60ef93f8316edfb4e8f8b5cd 2024-11-13T13:48:13,080 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:13,080 INFO [RS:0;0b62828576b1:44471 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T13:48:13,080 INFO [RS:0;0b62828576b1:44471 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T13:48:13,080 INFO [RS:0;0b62828576b1:44471 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T13:48:13,080 INFO [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T13:48:13,080 INFO [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T13:48:13,080 DEBUG [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T13:48:13,080 DEBUG [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T13:48:13,080 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T13:48:13,080 DEBUG [StoreFileCloser-30c9fe9b60ef93f8316edfb4e8f8b5cd-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/family/d45353417cd5452d858c26d97ce53f3b 2024-11-13T13:48:13,080 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T13:48:13,080 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T13:48:13,080 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T13:48:13,080 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T13:48:13,080 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T13:48:13,081 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-13T13:48:13,086 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/default/testBlockEvictionOnGracefulStop1731505690260/30c9fe9b60ef93f8316edfb4e8f8b5cd/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-13T13:48:13,086 INFO [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:13,086 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 30c9fe9b60ef93f8316edfb4e8f8b5cd: Waiting for close lock at 1731505693079Running coprocessor pre-close hooks at 1731505693079Disabling compacts and flushes for region at 1731505693079Disabling writes for close at 1731505693079Writing region close event to WAL at 1731505693081 (+2 ms)Running coprocessor post-close hooks at 1731505693086 (+5 ms)Closed at 1731505693086 2024-11-13T13:48:13,086 DEBUG [RS_CLOSE_REGION-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd. 2024-11-13T13:48:13,092 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d6e8403e259d4297929b251b2426b185 2024-11-13T13:48:13,106 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/info/d6e8403e259d4297929b251b2426b185 is 209, key is testBlockEvictionOnGracefulStop1731505690260,,1731505690266.30c9fe9b60ef93f8316edfb4e8f8b5cd./info:regioninfo/1731505693017/Put/seqid=0 2024-11-13T13:48:13,107 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/info/d6e8403e259d4297929b251b2426b185 2024-11-13T13:48:13,107 INFO [regionserver/0b62828576b1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T13:48:13,109 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/.tmp/info/d6e8403e259d4297929b251b2426b185 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/d6e8403e259d4297929b251b2426b185 2024-11-13T13:48:13,110 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/d6e8403e259d4297929b251b2426b185, delay=941 ms 2024-11-13T13:48:13,111 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/d6e8403e259d4297929b251b2426b185, entries=8, sequenceid=19, filesize=6.6 K 2024-11-13T13:48:13,112 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false 2024-11-13T13:48:13,113 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/72b24f9dadaa4217bbde9650730f1ae4 2024-11-13T13:48:13,113 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/info/d6e8403e259d4297929b251b2426b185 2024-11-13T13:48:13,114 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/ns/6beaf43c521e48a591e0b1fd2a2d48a8 2024-11-13T13:48:13,114 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/table/e26d8068ec2c4f998e89466d185b6da5 2024-11-13T13:48:13,119 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-13T13:48:13,119 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T13:48:13,119 INFO [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T13:48:13,120 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731505693080Running coprocessor pre-close hooks at 1731505693080Disabling compacts and flushes for region at 1731505693080Disabling writes for close at 1731505693080Obtaining lock to block concurrent updates at 1731505693081 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731505693081Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731505693081Flushing stores of hbase:meta,,1.1588230740 at 1731505693082 (+1 ms)Flushing 1588230740/info: creating writer at 1731505693082Flushing 1588230740/info: appending metadata at 1731505693105 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731505693106 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54c59d6d: reopening flushed file at 1731505693109 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false at 1731505693112 (+3 ms)Writing region close event to WAL at 1731505693115 (+3 ms)Running coprocessor post-close hooks at 1731505693119 (+4 ms)Closed at 1731505693119 2024-11-13T13:48:13,120 DEBUG [RS_CLOSE_META-regionserver/0b62828576b1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T13:48:13,124 INFO [regionserver/0b62828576b1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T13:48:13,280 INFO [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(976): stopping server 0b62828576b1,45901,1731505692465; all regions closed. 2024-11-13T13:48:13,280 INFO [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(976): stopping server 0b62828576b1,44471,1731505689185; all regions closed. 2024-11-13T13:48:13,282 DEBUG [RS:2;0b62828576b1:45901 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs 2024-11-13T13:48:13,282 INFO [RS:2;0b62828576b1:45901 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b62828576b1%2C45901%2C1731505692465:(num 1731505692664) 2024-11-13T13:48:13,282 DEBUG [RS:2;0b62828576b1:45901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:13,282 INFO [RS:2;0b62828576b1:45901 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T13:48:13,282 INFO [RS:2;0b62828576b1:45901 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T13:48:13,282 DEBUG [RS:0;0b62828576b1:44471 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs 2024-11-13T13:48:13,282 INFO [RS:2;0b62828576b1:45901 {}] hbase.ChoreService(370): Chore service for: regionserver/0b62828576b1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T13:48:13,282 INFO [RS:0;0b62828576b1:44471 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b62828576b1%2C44471%2C1731505689185.meta:.meta(num 1731505692511) 2024-11-13T13:48:13,283 INFO [RS:2;0b62828576b1:45901 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T13:48:13,283 INFO [regionserver/0b62828576b1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T13:48:13,283 INFO [RS:2;0b62828576b1:45901 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T13:48:13,283 INFO [RS:2;0b62828576b1:45901 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T13:48:13,283 INFO [RS:2;0b62828576b1:45901 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T13:48:13,283 INFO [RS:2;0b62828576b1:45901 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45901 2024-11-13T13:48:13,285 DEBUG [RS:0;0b62828576b1:44471 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/oldWALs 2024-11-13T13:48:13,285 INFO [RS:0;0b62828576b1:44471 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b62828576b1%2C44471%2C1731505689185:(num 1731505689744) 2024-11-13T13:48:13,285 DEBUG [RS:0;0b62828576b1:44471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T13:48:13,285 INFO [RS:0;0b62828576b1:44471 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T13:48:13,285 INFO [RS:0;0b62828576b1:44471 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T13:48:13,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T13:48:13,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x10039661cbf0003, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b62828576b1,45901,1731505692465 2024-11-13T13:48:13,285 INFO [RS:0;0b62828576b1:44471 {}] hbase.ChoreService(370): Chore service for: regionserver/0b62828576b1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T13:48:13,286 INFO [RS:0;0b62828576b1:44471 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T13:48:13,286 INFO [RS:2;0b62828576b1:45901 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T13:48:13,286 INFO [RS:0;0b62828576b1:44471 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44471 2024-11-13T13:48:13,287 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b62828576b1,45901,1731505692465] 2024-11-13T13:48:13,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b62828576b1,44471,1731505689185 2024-11-13T13:48:13,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T13:48:13,288 INFO [RS:0;0b62828576b1:44471 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T13:48:13,289 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b62828576b1,45901,1731505692465 already deleted, retry=false 2024-11-13T13:48:13,289 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b62828576b1,45901,1731505692465 expired; onlineServers=1 2024-11-13T13:48:13,290 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b62828576b1,44471,1731505689185] 2024-11-13T13:48:13,292 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b62828576b1,44471,1731505689185 already deleted, retry=false 2024-11-13T13:48:13,292 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b62828576b1,44471,1731505689185 expired; onlineServers=0 2024-11-13T13:48:13,292 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0b62828576b1,34663,1731505689137' ***** 2024-11-13T13:48:13,292 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T13:48:13,292 INFO [M:0;0b62828576b1:34663 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T13:48:13,292 INFO [M:0;0b62828576b1:34663 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T13:48:13,292 DEBUG [M:0;0b62828576b1:34663 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T13:48:13,292 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T13:48:13,292 DEBUG [M:0;0b62828576b1:34663 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T13:48:13,292 DEBUG [master/0b62828576b1:0:becomeActiveMaster-HFileCleaner.large.0-1731505689462 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b62828576b1:0:becomeActiveMaster-HFileCleaner.large.0-1731505689462,5,FailOnTimeoutGroup] 2024-11-13T13:48:13,292 DEBUG [master/0b62828576b1:0:becomeActiveMaster-HFileCleaner.small.0-1731505689465 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b62828576b1:0:becomeActiveMaster-HFileCleaner.small.0-1731505689465,5,FailOnTimeoutGroup] 2024-11-13T13:48:13,292 INFO [regionserver/0b62828576b1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T13:48:13,292 INFO [M:0;0b62828576b1:34663 {}] hbase.ChoreService(370): Chore service for: master/0b62828576b1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T13:48:13,292 INFO [M:0;0b62828576b1:34663 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T13:48:13,293 DEBUG [M:0;0b62828576b1:34663 {}] master.HMaster(1795): Stopping service threads 2024-11-13T13:48:13,293 INFO [M:0;0b62828576b1:34663 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T13:48:13,293 INFO [M:0;0b62828576b1:34663 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T13:48:13,293 INFO [M:0;0b62828576b1:34663 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T13:48:13,293 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T13:48:13,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T13:48:13,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T13:48:13,294 DEBUG [M:0;0b62828576b1:34663 {}] zookeeper.ZKUtil(347): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T13:48:13,294 WARN [M:0;0b62828576b1:34663 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T13:48:13,294 INFO [M:0;0b62828576b1:34663 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/.lastflushedseqids 2024-11-13T13:48:13,298 INFO [M:0;0b62828576b1:34663 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T13:48:13,298 INFO [M:0;0b62828576b1:34663 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T13:48:13,298 DEBUG [M:0;0b62828576b1:34663 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T13:48:13,298 INFO [M:0;0b62828576b1:34663 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:13,298 DEBUG [M:0;0b62828576b1:34663 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:13,298 DEBUG [M:0;0b62828576b1:34663 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T13:48:13,298 DEBUG [M:0;0b62828576b1:34663 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:13,298 INFO [M:0;0b62828576b1:34663 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-13T13:48:13,309 DEBUG [M:0;0b62828576b1:34663 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0951c7dd8ead4cc592c8a9cc668e577c 2024-11-13T13:48:13,324 DEBUG [M:0;0b62828576b1:34663 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0951c7dd8ead4cc592c8a9cc668e577c is 82, key is hbase:meta,,1/info:regioninfo/1731505692568/Put/seqid=0 2024-11-13T13:48:13,324 INFO [M:0;0b62828576b1:34663 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0951c7dd8ead4cc592c8a9cc668e577c 2024-11-13T13:48:13,333 DEBUG [M:0;0b62828576b1:34663 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 202c9c74b5024c6fab189318b499a0e1 2024-11-13T13:48:13,348 DEBUG [M:0;0b62828576b1:34663 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/202c9c74b5024c6fab189318b499a0e1 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731505690661/Put/seqid=0 2024-11-13T13:48:13,349 INFO [M:0;0b62828576b1:34663 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/202c9c74b5024c6fab189318b499a0e1 2024-11-13T13:48:13,358 DEBUG [M:0;0b62828576b1:34663 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 95dfbfd6d9174b91ba6e9e0b59ebaed0 2024-11-13T13:48:13,373 DEBUG [M:0;0b62828576b1:34663 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/95dfbfd6d9174b91ba6e9e0b59ebaed0 is 69, key is 0b62828576b1,44471,1731505689185/rs:state/1731505689574/Put/seqid=0 2024-11-13T13:48:13,374 INFO [M:0;0b62828576b1:34663 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/95dfbfd6d9174b91ba6e9e0b59ebaed0 2024-11-13T13:48:13,376 INFO [M:0;0b62828576b1:34663 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 95dfbfd6d9174b91ba6e9e0b59ebaed0 2024-11-13T13:48:13,376 DEBUG [M:0;0b62828576b1:34663 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0951c7dd8ead4cc592c8a9cc668e577c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0951c7dd8ead4cc592c8a9cc668e577c 2024-11-13T13:48:13,378 INFO [M:0;0b62828576b1:34663 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0951c7dd8ead4cc592c8a9cc668e577c, entries=14, sequenceid=102, filesize=5.9 K 2024-11-13T13:48:13,379 DEBUG [M:0;0b62828576b1:34663 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/202c9c74b5024c6fab189318b499a0e1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/202c9c74b5024c6fab189318b499a0e1 2024-11-13T13:48:13,380 INFO [M:0;0b62828576b1:34663 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/202c9c74b5024c6fab189318b499a0e1, entries=11, sequenceid=102, filesize=6.5 K 2024-11-13T13:48:13,381 DEBUG [M:0;0b62828576b1:34663 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/95dfbfd6d9174b91ba6e9e0b59ebaed0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/95dfbfd6d9174b91ba6e9e0b59ebaed0 2024-11-13T13:48:13,382 INFO [M:0;0b62828576b1:34663 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 95dfbfd6d9174b91ba6e9e0b59ebaed0 2024-11-13T13:48:13,382 INFO [M:0;0b62828576b1:34663 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/0ea81a0f-c683-b7c1-58b5-cae9501700f4/5ca04256-70be-4c37-d0c8-bf0953a71ff2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/95dfbfd6d9174b91ba6e9e0b59ebaed0, entries=3, sequenceid=102, filesize=5.3 K 2024-11-13T13:48:13,383 INFO [M:0;0b62828576b1:34663 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=102, compaction requested=false 2024-11-13T13:48:13,385 INFO [M:0;0b62828576b1:34663 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T13:48:13,385 DEBUG [M:0;0b62828576b1:34663 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731505693298Disabling compacts and flushes for region at 1731505693298Disabling writes for close at 1731505693298Obtaining lock to block concurrent updates at 1731505693298Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731505693298Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1731505693299 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731505693299Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731505693299Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731505693323 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731505693323Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731505693326 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731505693347 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731505693348 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731505693351 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731505693372 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731505693372Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61c9d3c7: reopening flushed file at 1731505693376 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24702941: reopening flushed file at 1731505693378 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52779b4d: reopening flushed file at 1731505693380 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=102, compaction requested=false at 1731505693383 (+3 ms)Writing region close event to WAL at 1731505693385 (+2 ms)Closed at 1731505693385 2024-11-13T13:48:13,386 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T13:48:13,386 INFO [M:0;0b62828576b1:34663 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T13:48:13,386 INFO [M:0;0b62828576b1:34663 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34663 2024-11-13T13:48:13,387 INFO [M:0;0b62828576b1:34663 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T13:48:13,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x10039661cbf0003, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:13,389 INFO [RS:2;0b62828576b1:45901 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T13:48:13,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x10039661cbf0003, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:13,389 INFO [RS:2;0b62828576b1:45901 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b62828576b1,45901,1731505692465; zookeeper connection closed. 2024-11-13T13:48:13,389 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@45f6a36c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@45f6a36c 2024-11-13T13:48:13,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:13,390 INFO [RS:0;0b62828576b1:44471 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T13:48:13,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44471-0x10039661cbf0001, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:13,390 INFO [RS:0;0b62828576b1:44471 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b62828576b1,44471,1731505689185; zookeeper connection closed. 2024-11-13T13:48:13,391 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ddef435 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ddef435 2024-11-13T13:48:13,391 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T13:48:13,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:13,489 INFO [M:0;0b62828576b1:34663 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T13:48:13,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x10039661cbf0000, quorum=127.0.0.1:58953, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T13:48:13,494 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T13:48:13,495 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T13:48:13,496 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T13:48:13,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=545 (was 575), ProcessCount=11 (was 11), AvailableMemoryMB=2701 (was 2795)