2024-11-08 23:49:06,091 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-08 23:49:06,110 main DEBUG Took 0.017419 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 23:49:06,111 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 23:49:06,111 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 23:49:06,112 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 23:49:06,113 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,121 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 23:49:06,135 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,137 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,138 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,138 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,139 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,139 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,140 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,140 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,141 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,142 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,144 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,145 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,145 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,145 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,146 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,147 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,147 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 23:49:06,148 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,148 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 23:49:06,150 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 23:49:06,151 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 23:49:06,152 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 23:49:06,153 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 23:49:06,154 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 23:49:06,154 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 23:49:06,162 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 23:49:06,164 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 23:49:06,166 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 23:49:06,166 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 23:49:06,167 main DEBUG createAppenders(={Console}) 2024-11-08 23:49:06,167 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-08 23:49:06,167 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-08 23:49:06,168 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-08 23:49:06,168 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 23:49:06,168 main DEBUG OutputStream closed 2024-11-08 23:49:06,169 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 23:49:06,169 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 23:49:06,169 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-08 23:49:06,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 23:49:06,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 23:49:06,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 23:49:06,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 23:49:06,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 23:49:06,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 23:49:06,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 23:49:06,238 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 23:49:06,238 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 23:49:06,238 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 23:49:06,238 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 23:49:06,239 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 23:49:06,239 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 23:49:06,239 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 23:49:06,239 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 23:49:06,240 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 23:49:06,240 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 23:49:06,241 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 23:49:06,243 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 23:49:06,243 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-08 23:49:06,244 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 23:49:06,244 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-08T23:49:06,258 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-08 23:49:06,261 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 23:49:06,261 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T23:49:06,478 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574 2024-11-08T23:49:06,501 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1871, ProcessCount=11, AvailableMemoryMB=3666 2024-11-08T23:49:06,590 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T23:49:06,674 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:06,688 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/cluster_9d5c141c-9ce4-ba07-7baf-54449ee0ced9, deleteOnExit=true 2024-11-08T23:49:06,781 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/cluster_9d5c141c-9ce4-ba07-7baf-54449ee0ced9/zookeeper_0, clientPort=60153, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/cluster_9d5c141c-9ce4-ba07-7baf-54449ee0ced9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/cluster_9d5c141c-9ce4-ba07-7baf-54449ee0ced9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T23:49:06,793 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60153 2024-11-08T23:49:06,794 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:06,883 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59 with version=8 2024-11-08T23:49:06,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/hbase-staging 2024-11-08T23:49:06,992 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T23:49:07,260 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a15bf3ddec7f:0 server-side Connection retries=45 2024-11-08T23:49:07,270 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:07,272 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:07,283 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T23:49:07,283 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:07,283 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T23:49:07,440 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T23:49:07,527 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T23:49:07,566 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T23:49:07,576 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T23:49:07,580 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T23:49:07,623 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12885 (auto-detected) 2024-11-08T23:49:07,624 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T23:49:07,646 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41931 2024-11-08T23:49:07,670 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41931 connecting to ZooKeeper ensemble=127.0.0.1:60153 2024-11-08T23:49:07,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:419310x0, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T23:49:07,711 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41931-0x100f9fc62d50000 connected 2024-11-08T23:49:07,738 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:07,739 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:07,746 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T23:49:07,751 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59, hbase.cluster.distributed=false 2024-11-08T23:49:07,778 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T23:49:07,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41931 2024-11-08T23:49:07,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41931 2024-11-08T23:49:07,784 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41931 2024-11-08T23:49:07,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41931 2024-11-08T23:49:07,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41931 2024-11-08T23:49:07,884 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a15bf3ddec7f:0 server-side Connection retries=45 2024-11-08T23:49:07,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:07,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:07,887 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T23:49:07,887 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:07,887 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T23:49:07,890 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T23:49:07,892 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T23:49:07,894 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33665 2024-11-08T23:49:07,896 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33665 connecting to ZooKeeper ensemble=127.0.0.1:60153 2024-11-08T23:49:07,896 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:07,897 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:07,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:336650x0, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T23:49:07,902 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33665-0x100f9fc62d50001 connected 2024-11-08T23:49:07,902 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T23:49:07,908 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T23:49:07,922 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/bucket.cache 2024-11-08T23:49:07,931 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T23:49:07,932 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T23:49:07,934 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T23:49:07,936 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T23:49:07,937 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T23:49:07,943 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T23:49:07,948 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T23:49:07,948 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/bucket.persistence is complete. 2024-11-08T23:49:07,950 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T23:49:07,953 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T23:49:07,958 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T23:49:07,960 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33665 2024-11-08T23:49:07,961 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33665 2024-11-08T23:49:07,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33665 2024-11-08T23:49:07,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33665 2024-11-08T23:49:07,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33665 2024-11-08T23:49:07,983 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a15bf3ddec7f:0 server-side Connection retries=45 2024-11-08T23:49:07,983 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:07,983 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:07,983 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T23:49:07,983 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:07,984 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T23:49:07,984 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T23:49:07,984 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T23:49:07,985 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44283 2024-11-08T23:49:07,987 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44283 connecting to ZooKeeper ensemble=127.0.0.1:60153 2024-11-08T23:49:07,987 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:07,988 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:07,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442830x0, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T23:49:07,993 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:442830x0, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T23:49:07,993 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44283-0x100f9fc62d50002 connected 2024-11-08T23:49:07,994 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T23:49:07,996 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/bucket.cache 2024-11-08T23:49:07,997 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T23:49:07,997 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T23:49:07,997 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T23:49:07,998 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T23:49:07,998 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T23:49:07,998 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T23:49:07,999 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T23:49:07,999 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/bucket.persistence is complete. 2024-11-08T23:49:07,999 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T23:49:08,001 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T23:49:08,003 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T23:49:08,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44283 2024-11-08T23:49:08,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44283 2024-11-08T23:49:08,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44283 2024-11-08T23:49:08,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44283 2024-11-08T23:49:08,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44283 2024-11-08T23:49:08,029 DEBUG [M:0;a15bf3ddec7f:41931 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a15bf3ddec7f:41931 2024-11-08T23:49:08,030 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a15bf3ddec7f,41931,1731109747063 2024-11-08T23:49:08,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T23:49:08,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T23:49:08,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T23:49:08,041 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a15bf3ddec7f,41931,1731109747063 2024-11-08T23:49:08,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T23:49:08,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T23:49:08,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,066 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T23:49:08,067 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a15bf3ddec7f,41931,1731109747063 from backup master directory 2024-11-08T23:49:08,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T23:49:08,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T23:49:08,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a15bf3ddec7f,41931,1731109747063 2024-11-08T23:49:08,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T23:49:08,072 WARN [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T23:49:08,072 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a15bf3ddec7f,41931,1731109747063 2024-11-08T23:49:08,074 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T23:49:08,076 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T23:49:08,083 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/hbase.id] with ID: 96966df6-62f1-4e46-bd59-6593e1d2139e 2024-11-08T23:49:08,083 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/.tmp/hbase.id 2024-11-08T23:49:08,090 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/hbase.id] 2024-11-08T23:49:08,135 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:08,140 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T23:49:08,159 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-08T23:49:08,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,196 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T23:49:08,200 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T23:49:08,252 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T23:49:08,260 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T23:49:08,328 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store 2024-11-08T23:49:08,352 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T23:49:08,355 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:08,357 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T23:49:08,357 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:08,357 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:08,359 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T23:49:08,359 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:08,359 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:08,360 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731109748356Disabling compacts and flushes for region at 1731109748356Disabling writes for close at 1731109748359 (+3 ms)Writing region close event to WAL at 1731109748359Closed at 1731109748359 2024-11-08T23:49:08,360 WARN [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/.initializing 2024-11-08T23:49:08,360 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/WALs/a15bf3ddec7f,41931,1731109747063 2024-11-08T23:49:08,374 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T23:49:08,387 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a15bf3ddec7f%2C41931%2C1731109747063, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/WALs/a15bf3ddec7f,41931,1731109747063, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/oldWALs, maxLogs=10 2024-11-08T23:49:08,426 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/WALs/a15bf3ddec7f,41931,1731109747063/a15bf3ddec7f%2C41931%2C1731109747063.1731109748392 2024-11-08T23:49:08,427 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T23:49:08,427 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T23:49:08,428 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:08,431 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:08,432 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:08,481 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:08,513 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T23:49:08,518 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:08,519 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:08,520 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:08,525 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T23:49:08,526 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:08,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T23:49:08,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:08,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T23:49:08,531 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:08,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T23:49:08,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:08,537 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T23:49:08,538 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:08,538 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T23:49:08,539 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:08,542 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:08,542 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:08,545 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:08,546 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:08,549 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T23:49:08,550 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:08,557 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T23:49:08,558 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61439088, jitterRate=-0.08448624610900879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T23:49:08,564 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731109748445Initializing all the Stores at 1731109748446 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109748446Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109748447 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109748447Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109748447Cleaning up temporary data from old regions at 1731109748546 (+99 ms)Region opened successfully at 1731109748564 (+18 ms) 2024-11-08T23:49:08,566 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T23:49:08,598 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54383089, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a15bf3ddec7f/172.17.0.2:0 2024-11-08T23:49:08,632 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T23:49:08,645 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T23:49:08,645 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T23:49:08,649 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T23:49:08,649 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T23:49:08,656 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T23:49:08,657 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T23:49:08,686 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T23:49:08,695 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T23:49:08,698 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T23:49:08,701 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T23:49:08,703 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T23:49:08,705 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T23:49:08,707 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T23:49:08,711 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T23:49:08,713 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T23:49:08,715 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T23:49:08,717 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T23:49:08,743 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T23:49:08,747 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T23:49:08,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T23:49:08,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T23:49:08,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T23:49:08,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,767 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a15bf3ddec7f,41931,1731109747063, sessionid=0x100f9fc62d50000, setting cluster-up flag (Was=false) 2024-11-08T23:49:08,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,790 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T23:49:08,792 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a15bf3ddec7f,41931,1731109747063 2024-11-08T23:49:08,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:08,803 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T23:49:08,804 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a15bf3ddec7f,41931,1731109747063 2024-11-08T23:49:08,809 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T23:49:08,817 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(746): ClusterId : 96966df6-62f1-4e46-bd59-6593e1d2139e 2024-11-08T23:49:08,817 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(746): ClusterId : 96966df6-62f1-4e46-bd59-6593e1d2139e 2024-11-08T23:49:08,819 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T23:49:08,819 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T23:49:08,826 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T23:49:08,826 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T23:49:08,827 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T23:49:08,827 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T23:49:08,830 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T23:49:08,831 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T23:49:08,831 DEBUG [RS:0;a15bf3ddec7f:33665 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@134035e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a15bf3ddec7f/172.17.0.2:0 2024-11-08T23:49:08,831 DEBUG [RS:1;a15bf3ddec7f:44283 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e31500c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a15bf3ddec7f/172.17.0.2:0 2024-11-08T23:49:08,847 DEBUG [RS:0;a15bf3ddec7f:33665 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a15bf3ddec7f:33665 2024-11-08T23:49:08,850 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T23:49:08,850 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T23:49:08,850 DEBUG [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T23:49:08,853 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(2659): reportForDuty to master=a15bf3ddec7f,41931,1731109747063 with port=33665, startcode=1731109747845 2024-11-08T23:49:08,853 DEBUG [RS:1;a15bf3ddec7f:44283 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a15bf3ddec7f:44283 2024-11-08T23:49:08,853 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T23:49:08,853 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T23:49:08,853 DEBUG [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T23:49:08,855 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(2659): reportForDuty to master=a15bf3ddec7f,41931,1731109747063 with port=44283, startcode=1731109747982 2024-11-08T23:49:08,866 DEBUG [RS:0;a15bf3ddec7f:33665 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T23:49:08,866 DEBUG [RS:1;a15bf3ddec7f:44283 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T23:49:08,892 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T23:49:08,904 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T23:49:08,918 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T23:49:08,925 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a15bf3ddec7f,41931,1731109747063 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T23:49:08,935 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a15bf3ddec7f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T23:49:08,936 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a15bf3ddec7f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T23:49:08,936 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T23:49:08,936 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T23:49:08,937 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a15bf3ddec7f:0, corePoolSize=10, maxPoolSize=10 2024-11-08T23:49:08,937 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:08,937 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T23:49:08,937 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:08,942 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109778942 2024-11-08T23:49:08,943 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41993, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T23:49:08,943 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46193, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T23:49:08,944 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T23:49:08,945 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T23:49:08,945 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T23:49:08,945 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T23:49:08,949 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T23:49:08,950 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T23:49:08,950 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T23:49:08,950 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T23:49:08,951 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:08,951 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41931 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T23:49:08,951 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T23:49:08,952 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:08,956 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T23:49:08,958 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T23:49:08,958 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T23:49:08,959 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41931 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T23:49:08,965 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T23:49:08,965 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T23:49:08,968 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.large.0-1731109748967,5,FailOnTimeoutGroup] 2024-11-08T23:49:08,969 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.small.0-1731109748969,5,FailOnTimeoutGroup] 2024-11-08T23:49:08,969 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:08,969 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T23:49:08,970 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:08,971 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:08,989 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T23:49:08,989 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59 2024-11-08T23:49:08,991 DEBUG [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-08T23:49:08,991 DEBUG [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-08T23:49:08,991 WARN [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-08T23:49:08,991 WARN [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-08T23:49:09,001 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:09,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T23:49:09,007 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T23:49:09,008 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:09,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:09,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T23:49:09,012 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T23:49:09,013 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:09,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:09,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T23:49:09,017 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T23:49:09,017 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:09,018 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:09,018 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T23:49:09,022 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T23:49:09,022 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:09,022 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:09,023 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T23:49:09,023 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740 2024-11-08T23:49:09,024 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740 2024-11-08T23:49:09,024 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T23:49:09,024 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T23:49:09,025 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T23:49:09,025 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T23:49:09,033 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T23:49:09,033 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60543054, jitterRate=-0.09783819317817688}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T23:49:09,036 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731109749001Initializing all the Stores at 1731109749001Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109749002 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109749002Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109749002Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109749002Cleaning up temporary data from old regions at 1731109749024 (+22 ms)Region opened successfully at 1731109749036 (+12 ms) 2024-11-08T23:49:09,036 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T23:49:09,037 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T23:49:09,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T23:49:09,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T23:49:09,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T23:49:09,039 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T23:49:09,039 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731109749036Disabling compacts and flushes for region at 1731109749036Disabling writes for close at 1731109749037 (+1 ms)Writing region close event to WAL at 1731109749038 (+1 ms)Closed at 1731109749039 (+1 ms) 2024-11-08T23:49:09,043 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T23:49:09,043 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T23:49:09,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T23:49:09,058 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T23:49:09,060 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T23:49:09,093 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(2659): reportForDuty to master=a15bf3ddec7f,41931,1731109747063 with port=33665, startcode=1731109747845 2024-11-08T23:49:09,093 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(2659): reportForDuty to master=a15bf3ddec7f,41931,1731109747063 with port=44283, startcode=1731109747982 2024-11-08T23:49:09,095 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41931 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a15bf3ddec7f,44283,1731109747982 2024-11-08T23:49:09,098 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41931 {}] master.ServerManager(517): Registering regionserver=a15bf3ddec7f,44283,1731109747982 2024-11-08T23:49:09,105 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41931 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:09,105 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41931 {}] master.ServerManager(517): Registering regionserver=a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:09,106 DEBUG [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59 2024-11-08T23:49:09,106 DEBUG [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T23:49:09,106 DEBUG [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T23:49:09,107 DEBUG [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59 2024-11-08T23:49:09,107 DEBUG [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T23:49:09,107 DEBUG [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T23:49:09,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T23:49:09,115 DEBUG [RS:0;a15bf3ddec7f:33665 {}] zookeeper.ZKUtil(111): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:09,115 WARN [RS:0;a15bf3ddec7f:33665 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T23:49:09,115 DEBUG [RS:1;a15bf3ddec7f:44283 {}] zookeeper.ZKUtil(111): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a15bf3ddec7f,44283,1731109747982 2024-11-08T23:49:09,115 INFO [RS:0;a15bf3ddec7f:33665 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T23:49:09,115 WARN [RS:1;a15bf3ddec7f:44283 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T23:49:09,116 INFO [RS:1;a15bf3ddec7f:44283 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T23:49:09,116 DEBUG [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:09,116 DEBUG [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,44283,1731109747982 2024-11-08T23:49:09,118 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a15bf3ddec7f,33665,1731109747845] 2024-11-08T23:49:09,118 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a15bf3ddec7f,44283,1731109747982] 2024-11-08T23:49:09,152 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T23:49:09,152 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T23:49:09,166 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T23:49:09,166 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T23:49:09,171 INFO [RS:0;a15bf3ddec7f:33665 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T23:49:09,171 INFO [RS:1;a15bf3ddec7f:44283 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T23:49:09,172 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,172 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,173 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T23:49:09,173 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T23:49:09,178 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T23:49:09,178 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T23:49:09,179 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,179 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,180 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,180 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,180 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,180 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,180 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,180 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,180 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,180 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,180 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,181 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a15bf3ddec7f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T23:49:09,181 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,181 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,181 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a15bf3ddec7f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T23:49:09,181 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,181 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,181 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,181 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,181 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,181 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,182 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,182 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,182 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a15bf3ddec7f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T23:49:09,182 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,182 DEBUG [RS:0;a15bf3ddec7f:33665 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a15bf3ddec7f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T23:49:09,182 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,182 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:09,183 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a15bf3ddec7f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T23:49:09,183 DEBUG [RS:1;a15bf3ddec7f:44283 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a15bf3ddec7f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T23:49:09,185 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,185 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,185 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,185 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,186 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,186 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,186 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,186 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,186 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,186 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,33665,1731109747845-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T23:49:09,186 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,186 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,44283,1731109747982-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T23:49:09,212 WARN [a15bf3ddec7f:41931 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T23:49:09,214 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T23:49:09,214 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T23:49:09,216 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,44283,1731109747982-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,216 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,33665,1731109747845-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,216 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,216 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,217 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.Replication(171): a15bf3ddec7f,33665,1731109747845 started 2024-11-08T23:49:09,217 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.Replication(171): a15bf3ddec7f,44283,1731109747982 started 2024-11-08T23:49:09,245 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,245 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,246 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(1482): Serving as a15bf3ddec7f,44283,1731109747982, RpcServer on a15bf3ddec7f/172.17.0.2:44283, sessionid=0x100f9fc62d50002 2024-11-08T23:49:09,246 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(1482): Serving as a15bf3ddec7f,33665,1731109747845, RpcServer on a15bf3ddec7f/172.17.0.2:33665, sessionid=0x100f9fc62d50001 2024-11-08T23:49:09,247 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T23:49:09,247 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T23:49:09,247 DEBUG [RS:1;a15bf3ddec7f:44283 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a15bf3ddec7f,44283,1731109747982 2024-11-08T23:49:09,247 DEBUG [RS:0;a15bf3ddec7f:33665 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:09,247 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a15bf3ddec7f,44283,1731109747982' 2024-11-08T23:49:09,247 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a15bf3ddec7f,33665,1731109747845' 2024-11-08T23:49:09,248 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T23:49:09,248 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T23:49:09,249 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T23:49:09,249 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T23:49:09,250 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T23:49:09,250 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T23:49:09,250 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T23:49:09,250 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T23:49:09,250 DEBUG [RS:0;a15bf3ddec7f:33665 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:09,250 DEBUG [RS:1;a15bf3ddec7f:44283 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a15bf3ddec7f,44283,1731109747982 2024-11-08T23:49:09,250 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a15bf3ddec7f,44283,1731109747982' 2024-11-08T23:49:09,250 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a15bf3ddec7f,33665,1731109747845' 2024-11-08T23:49:09,250 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T23:49:09,250 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T23:49:09,251 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T23:49:09,251 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T23:49:09,251 DEBUG [RS:0;a15bf3ddec7f:33665 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T23:49:09,252 DEBUG [RS:1;a15bf3ddec7f:44283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T23:49:09,252 INFO [RS:0;a15bf3ddec7f:33665 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T23:49:09,252 INFO [RS:1;a15bf3ddec7f:44283 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T23:49:09,252 INFO [RS:0;a15bf3ddec7f:33665 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T23:49:09,252 INFO [RS:1;a15bf3ddec7f:44283 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T23:49:09,357 INFO [RS:0;a15bf3ddec7f:33665 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T23:49:09,357 INFO [RS:1;a15bf3ddec7f:44283 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T23:49:09,358 INFO [RS:1;a15bf3ddec7f:44283 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a15bf3ddec7f%2C44283%2C1731109747982, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,44283,1731109747982, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs, maxLogs=55 2024-11-08T23:49:09,358 INFO [RS:0;a15bf3ddec7f:33665 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a15bf3ddec7f%2C33665%2C1731109747845, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,33665,1731109747845, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs, maxLogs=55 2024-11-08T23:49:09,381 INFO [RS:0;a15bf3ddec7f:33665 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,33665,1731109747845/a15bf3ddec7f%2C33665%2C1731109747845.1731109749363 2024-11-08T23:49:09,382 DEBUG [RS:0;a15bf3ddec7f:33665 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T23:49:09,384 INFO [RS:1;a15bf3ddec7f:44283 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,44283,1731109747982/a15bf3ddec7f%2C44283%2C1731109747982.1731109749363 2024-11-08T23:49:09,384 DEBUG [RS:1;a15bf3ddec7f:44283 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T23:49:09,464 DEBUG [a15bf3ddec7f:41931 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T23:49:09,476 DEBUG [a15bf3ddec7f:41931 {}] balancer.BalancerClusterState(204): Hosts are {a15bf3ddec7f=0} racks are {/default-rack=0} 2024-11-08T23:49:09,479 DEBUG [a15bf3ddec7f:41931 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T23:49:09,479 DEBUG [a15bf3ddec7f:41931 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T23:49:09,479 DEBUG [a15bf3ddec7f:41931 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T23:49:09,479 DEBUG [a15bf3ddec7f:41931 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T23:49:09,479 INFO [a15bf3ddec7f:41931 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T23:49:09,480 INFO [a15bf3ddec7f:41931 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T23:49:09,480 DEBUG [a15bf3ddec7f:41931 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T23:49:09,487 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:09,493 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,33665,1731109747845, state=OPENING 2024-11-08T23:49:09,499 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T23:49:09,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:09,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:09,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:09,502 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:09,503 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:09,503 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:09,504 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T23:49:09,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a15bf3ddec7f,33665,1731109747845}] 2024-11-08T23:49:09,679 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T23:49:09,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52023, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T23:49:09,690 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T23:49:09,691 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T23:49:09,692 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T23:49:09,693 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a15bf3ddec7f%2C33665%2C1731109747845.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,33665,1731109747845, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs, maxLogs=55 2024-11-08T23:49:09,714 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,33665,1731109747845/a15bf3ddec7f%2C33665%2C1731109747845.meta.1731109749694.meta 2024-11-08T23:49:09,715 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T23:49:09,715 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T23:49:09,717 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T23:49:09,720 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T23:49:09,725 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T23:49:09,729 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T23:49:09,729 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:09,730 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T23:49:09,730 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T23:49:09,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T23:49:09,732 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T23:49:09,732 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:09,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:09,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T23:49:09,733 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T23:49:09,734 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:09,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:09,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T23:49:09,735 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T23:49:09,735 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:09,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:09,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T23:49:09,736 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T23:49:09,736 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:09,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:09,737 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T23:49:09,737 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740 2024-11-08T23:49:09,738 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740 2024-11-08T23:49:09,739 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T23:49:09,739 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T23:49:09,739 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T23:49:09,740 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T23:49:09,741 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63744214, jitterRate=-0.05013719201087952}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T23:49:09,741 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T23:49:09,742 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731109749730Writing region info on filesystem at 1731109749730Initializing all the Stores at 1731109749731 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109749731Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109749731Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109749731Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109749731Cleaning up temporary data from old regions at 1731109749739 (+8 ms)Running coprocessor post-open hooks at 1731109749741 (+2 ms)Region opened successfully at 1731109749742 (+1 ms) 2024-11-08T23:49:09,748 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731109749671 2024-11-08T23:49:09,758 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T23:49:09,759 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T23:49:09,760 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:09,762 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,33665,1731109747845, state=OPEN 2024-11-08T23:49:09,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T23:49:09,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T23:49:09,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T23:49:09,765 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:09,765 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:09,765 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:09,765 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:09,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T23:49:09,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a15bf3ddec7f,33665,1731109747845 in 260 msec 2024-11-08T23:49:09,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T23:49:09,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 719 msec 2024-11-08T23:49:09,773 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T23:49:09,773 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T23:49:09,793 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T23:49:09,794 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a15bf3ddec7f,33665,1731109747845, seqNum=-1] 2024-11-08T23:49:09,813 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T23:49:09,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59709, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T23:49:09,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 992 msec 2024-11-08T23:49:09,836 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731109749835, completionTime=-1 2024-11-08T23:49:09,839 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-08T23:49:09,840 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T23:49:09,864 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T23:49:09,864 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109809864 2024-11-08T23:49:09,864 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109869864 2024-11-08T23:49:09,864 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-08T23:49:09,867 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T23:49:09,874 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,41931,1731109747063-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,874 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,41931,1731109747063-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,875 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,41931,1731109747063-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,876 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a15bf3ddec7f:41931, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,876 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,877 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,882 DEBUG [master/a15bf3ddec7f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T23:49:09,903 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.831sec 2024-11-08T23:49:09,904 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T23:49:09,905 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T23:49:09,906 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T23:49:09,907 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T23:49:09,907 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T23:49:09,908 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,41931,1731109747063-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T23:49:09,908 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,41931,1731109747063-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T23:49:09,910 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T23:49:09,910 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T23:49:09,911 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,41931,1731109747063-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:09,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218aa893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T23:49:09,929 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T23:49:09,929 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T23:49:09,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a15bf3ddec7f,41931,-1 for getting cluster id 2024-11-08T23:49:09,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T23:49:09,945 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '96966df6-62f1-4e46-bd59-6593e1d2139e' 2024-11-08T23:49:09,947 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T23:49:09,948 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "96966df6-62f1-4e46-bd59-6593e1d2139e" 2024-11-08T23:49:09,948 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b528420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T23:49:09,948 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a15bf3ddec7f,41931,-1] 2024-11-08T23:49:09,951 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T23:49:09,952 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:09,954 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T23:49:09,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f82a59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T23:49:09,957 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T23:49:09,964 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a15bf3ddec7f,33665,1731109747845, seqNum=-1] 2024-11-08T23:49:09,964 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T23:49:09,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T23:49:09,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a15bf3ddec7f,41931,1731109747063 2024-11-08T23:49:10,003 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T23:49:10,009 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a15bf3ddec7f,41931,1731109747063 2024-11-08T23:49:10,011 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@608fd6b2 2024-11-08T23:49:10,012 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T23:49:10,015 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T23:49:10,020 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41931 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731109749997', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T23:49:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41931 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731109749997 2024-11-08T23:49:10,029 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731109749997 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T23:49:10,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41931 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731109749997" procId is: 4 2024-11-08T23:49:10,031 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:10,033 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731109749997 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T23:49:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41931 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T23:49:10,047 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731109749997-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b47e8c6e0f01d522d8a759095cdb764d, NAME => 'testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731109749997', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59 2024-11-08T23:49:10,054 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731109749997-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:10,055 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731109749997-pool-0 {}] regionserver.HRegion(1722): Closing b47e8c6e0f01d522d8a759095cdb764d, disabling compactions & flushes 2024-11-08T23:49:10,055 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731109749997-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:10,055 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731109749997-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:10,055 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731109749997-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. after waiting 0 ms 2024-11-08T23:49:10,055 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731109749997-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:10,055 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731109749997-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:10,055 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731109749997-pool-0 {}] regionserver.HRegion(1676): Region close journal for b47e8c6e0f01d522d8a759095cdb764d: Waiting for close lock at 1731109750055Disabling compacts and flushes for region at 1731109750055Disabling writes for close at 1731109750055Writing region close event to WAL at 1731109750055Closed at 1731109750055 2024-11-08T23:49:10,057 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731109749997 execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T23:49:10,062 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731109750057"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731109750057"}]},"ts":"1731109750057"} 2024-11-08T23:49:10,066 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T23:49:10,067 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731109749997 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T23:49:10,070 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731109749997","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109750067"}]},"ts":"1731109750067"} 2024-11-08T23:49:10,073 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731109749997, state=ENABLING in hbase:meta 2024-11-08T23:49:10,074 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a15bf3ddec7f=0} racks are {/default-rack=0} 2024-11-08T23:49:10,074 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T23:49:10,074 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T23:49:10,074 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T23:49:10,074 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T23:49:10,075 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T23:49:10,075 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T23:49:10,075 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T23:49:10,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731109749997, region=b47e8c6e0f01d522d8a759095cdb764d, ASSIGN}] 2024-11-08T23:49:10,077 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731109749997, region=b47e8c6e0f01d522d8a759095cdb764d, ASSIGN 2024-11-08T23:49:10,078 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731109749997, region=b47e8c6e0f01d522d8a759095cdb764d, ASSIGN; state=OFFLINE, location=a15bf3ddec7f,44283,1731109747982; forceNewPlan=false, retain=false 2024-11-08T23:49:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41931 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T23:49:10,230 INFO [a15bf3ddec7f:41931 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T23:49:10,231 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b47e8c6e0f01d522d8a759095cdb764d, regionState=OPENING, regionLocation=a15bf3ddec7f,44283,1731109747982 2024-11-08T23:49:10,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731109749997, region=b47e8c6e0f01d522d8a759095cdb764d, ASSIGN because future has completed 2024-11-08T23:49:10,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b47e8c6e0f01d522d8a759095cdb764d, server=a15bf3ddec7f,44283,1731109747982}] 2024-11-08T23:49:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41931 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T23:49:10,389 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T23:49:10,391 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40409, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T23:49:10,393 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:10,393 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b47e8c6e0f01d522d8a759095cdb764d, NAME => 'testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T23:49:10,394 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731109749997 b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:10,394 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:10,394 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:10,394 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:10,395 INFO [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:10,400 INFO [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b47e8c6e0f01d522d8a759095cdb764d columnFamilyName family 2024-11-08T23:49:10,400 DEBUG [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:10,401 INFO [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] regionserver.HStore(327): Store=b47e8c6e0f01d522d8a759095cdb764d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T23:49:10,401 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:10,402 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:10,402 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:10,403 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:10,403 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:10,404 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:10,412 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T23:49:10,412 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b47e8c6e0f01d522d8a759095cdb764d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68292920, jitterRate=0.01764380931854248}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T23:49:10,412 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:10,414 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b47e8c6e0f01d522d8a759095cdb764d: Running coprocessor pre-open hook at 1731109750395Writing region info on filesystem at 1731109750395Initializing all the Stores at 1731109750395Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109750395Cleaning up temporary data from old regions at 1731109750403 (+8 ms)Running coprocessor post-open hooks at 1731109750412 (+9 ms)Region opened successfully at 1731109750414 (+2 ms) 2024-11-08T23:49:10,415 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d., pid=6, masterSystemTime=1731109750389 2024-11-08T23:49:10,418 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:10,418 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:10,418 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b47e8c6e0f01d522d8a759095cdb764d, regionState=OPEN, openSeqNum=2, regionLocation=a15bf3ddec7f,44283,1731109747982 2024-11-08T23:49:10,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b47e8c6e0f01d522d8a759095cdb764d, server=a15bf3ddec7f,44283,1731109747982 because future has completed 2024-11-08T23:49:10,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T23:49:10,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b47e8c6e0f01d522d8a759095cdb764d, server=a15bf3ddec7f,44283,1731109747982 in 187 msec 2024-11-08T23:49:10,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T23:49:10,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731109749997, region=b47e8c6e0f01d522d8a759095cdb764d, ASSIGN in 350 msec 2024-11-08T23:49:10,429 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731109749997 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T23:49:10,430 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731109749997","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109750429"}]},"ts":"1731109750429"} 2024-11-08T23:49:10,433 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731109749997, state=ENABLED in hbase:meta 2024-11-08T23:49:10,433 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731109749997 execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T23:49:10,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731109749997 in 410 msec 2024-11-08T23:49:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41931 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T23:49:10,672 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731109749997 get assigned. Timeout = 60000ms 2024-11-08T23:49:10,672 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731109749997 completed 2024-11-08T23:49:10,675 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T23:49:10,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731109749997 assigned to meta. Checking AM states. 2024-11-08T23:49:10,680 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T23:49:10,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731109749997 assigned. 2024-11-08T23:49:10,689 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731109749997', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d., hostname=a15bf3ddec7f,44283,1731109747982, seqNum=2] 2024-11-08T23:49:10,690 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T23:49:10,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T23:49:10,702 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b47e8c6e0f01d522d8a759095cdb764d 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-08T23:49:10,727 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f251d59e02064b3fbbe703acc71f7ef2 2024-11-08T23:49:10,766 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/.tmp/family/f251d59e02064b3fbbe703acc71f7ef2 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-08T23:49:10,771 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/.tmp/family/f251d59e02064b3fbbe703acc71f7ef2 2024-11-08T23:49:10,806 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/.tmp/family/f251d59e02064b3fbbe703acc71f7ef2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/family/f251d59e02064b3fbbe703acc71f7ef2 2024-11-08T23:49:10,808 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/family/f251d59e02064b3fbbe703acc71f7ef2, delay=940 ms 2024-11-08T23:49:10,809 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/family/f251d59e02064b3fbbe703acc71f7ef2, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T23:49:10,815 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for b47e8c6e0f01d522d8a759095cdb764d in 112ms, sequenceid=6, compaction requested=false 2024-11-08T23:49:10,816 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731109749997' 2024-11-08T23:49:10,818 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b47e8c6e0f01d522d8a759095cdb764d: 2024-11-08T23:49:11,750 DEBUG [hfile-prefetch-1731109750808 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f251d59e02064b3fbbe703acc71f7ef2_0. 2024-11-08T23:49:11,754 DEBUG [hfile-prefetch-1731109750808 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f251d59e02064b3fbbe703acc71f7ef2_117. 2024-11-08T23:49:11,755 DEBUG [hfile-prefetch-1731109750808 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/family/f251d59e02064b3fbbe703acc71f7ef2, with total blocks 2, and data blocks 1 2024-11-08T23:49:11,755 DEBUG [hfile-prefetch-1731109750808 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-08T23:49:11,755 DEBUG [hfile-prefetch-1731109750808 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/family/f251d59e02064b3fbbe703acc71f7ef2, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-08T23:49:11,756 DEBUG [hfile-prefetch-1731109750808 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/family/f251d59e02064b3fbbe703acc71f7ef2. Will sleep 100ms and try the verification again. 2024-11-08T23:49:11,801 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T23:49:11,801 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T23:49:11,801 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T23:49:11,802 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T23:49:11,821 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-08T23:49:11,821 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731109751801 2024-11-08T23:49:11,821 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T23:49:11,828 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41931 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=b47e8c6e0f01d522d8a759095cdb764d, source=a15bf3ddec7f,44283,1731109747982, destination=a15bf3ddec7f,33665,1731109747845, warming up region on a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:11,830 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41931 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=b47e8c6e0f01d522d8a759095cdb764d, source=a15bf3ddec7f,44283,1731109747982, destination=a15bf3ddec7f,33665,1731109747845, running balancer 2024-11-08T23:49:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41931 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731109749997, region=b47e8c6e0f01d522d8a759095cdb764d, REOPEN/MOVE 2024-11-08T23:49:11,831 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731109749997, region=b47e8c6e0f01d522d8a759095cdb764d, REOPEN/MOVE 2024-11-08T23:49:11,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33665 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33665 {}] regionserver.HRegion(7855): Warmup {ENCODED => b47e8c6e0f01d522d8a759095cdb764d, NAME => 'testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T23:49:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33665 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:11,833 INFO [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:11,833 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b47e8c6e0f01d522d8a759095cdb764d, regionState=CLOSING, regionLocation=a15bf3ddec7f,44283,1731109747982 2024-11-08T23:49:11,834 INFO [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b47e8c6e0f01d522d8a759095cdb764d columnFamilyName family 2024-11-08T23:49:11,834 DEBUG [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:11,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731109749997, region=b47e8c6e0f01d522d8a759095cdb764d, REOPEN/MOVE because future has completed 2024-11-08T23:49:11,837 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-08T23:49:11,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure b47e8c6e0f01d522d8a759095cdb764d, server=a15bf3ddec7f,44283,1731109747982}] 2024-11-08T23:49:11,838 DEBUG [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/family/f251d59e02064b3fbbe703acc71f7ef2 2024-11-08T23:49:11,838 INFO [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] regionserver.HStore(327): Store=b47e8c6e0f01d522d8a759095cdb764d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T23:49:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33665 {}] regionserver.HRegion(1722): Closing b47e8c6e0f01d522d8a759095cdb764d, disabling compactions & flushes 2024-11-08T23:49:11,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33665 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33665 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33665 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. after waiting 0 ms 2024-11-08T23:49:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33665 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:11,839 DEBUG [StoreFileCloser-b47e8c6e0f01d522d8a759095cdb764d-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/family/f251d59e02064b3fbbe703acc71f7ef2 2024-11-08T23:49:11,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33665 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33665 {}] regionserver.HRegion(1676): Region close journal for b47e8c6e0f01d522d8a759095cdb764d: Waiting for close lock at 1731109751838Disabling compacts and flushes for region at 1731109751838Disabling writes for close at 1731109751838Writing region close event to WAL at 1731109751840 (+2 ms)Closed at 1731109751840 2024-11-08T23:49:11,842 WARN [hfile-prefetch-1731109750808 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/family/f251d59e02064b3fbbe703acc71f7ef2, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-08T23:49:11,843 DEBUG [hfile-prefetch-1731109750808 {}] hfile.PrefetchExecutor(121): Prefetch completed for f251d59e02064b3fbbe703acc71f7ef2 2024-11-08T23:49:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41931 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-08T23:49:11,996 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:11,996 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-08T23:49:11,997 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing b47e8c6e0f01d522d8a759095cdb764d, disabling compactions & flushes 2024-11-08T23:49:11,997 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:11,997 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:11,997 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. after waiting 0 ms 2024-11-08T23:49:11,997 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:11,998 DEBUG [StoreFileCloser-b47e8c6e0f01d522d8a759095cdb764d-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file f251d59e02064b3fbbe703acc71f7ef2, starting offset: 0, end offset: 9223372036854775807 2024-11-08T23:49:12,000 DEBUG [StoreFileCloser-b47e8c6e0f01d522d8a759095cdb764d-family-1 {}] bucket.BucketCache(857): removed key f251d59e02064b3fbbe703acc71f7ef2_0 from back map with offset lock 2094080 in the evict process 2024-11-08T23:49:12,005 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T23:49:12,007 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:12,007 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for b47e8c6e0f01d522d8a759095cdb764d: Waiting for close lock at 1731109751997Running coprocessor pre-close hooks at 1731109751997Disabling compacts and flushes for region at 1731109751997Disabling writes for close at 1731109751997Writing region close event to WAL at 1731109752000 (+3 ms)Running coprocessor post-close hooks at 1731109752005 (+5 ms)Closed at 1731109752007 (+2 ms) 2024-11-08T23:49:12,007 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding b47e8c6e0f01d522d8a759095cdb764d move to a15bf3ddec7f,33665,1731109747845 record at close sequenceid=6 2024-11-08T23:49:12,010 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,011 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b47e8c6e0f01d522d8a759095cdb764d, regionState=CLOSED 2024-11-08T23:49:12,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure b47e8c6e0f01d522d8a759095cdb764d, server=a15bf3ddec7f,44283,1731109747982 because future has completed 2024-11-08T23:49:12,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T23:49:12,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure b47e8c6e0f01d522d8a759095cdb764d, server=a15bf3ddec7f,44283,1731109747982 in 176 msec 2024-11-08T23:49:12,017 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731109749997, region=b47e8c6e0f01d522d8a759095cdb764d, REOPEN/MOVE; state=CLOSED, location=a15bf3ddec7f,33665,1731109747845; forceNewPlan=false, retain=false 2024-11-08T23:49:12,021 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T23:49:12,021 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T23:49:12,021 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T23:49:12,021 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T23:49:12,025 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T23:49:12,025 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731109752021 2024-11-08T23:49:12,025 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T23:49:12,168 INFO [a15bf3ddec7f:41931 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T23:49:12,168 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b47e8c6e0f01d522d8a759095cdb764d, regionState=OPENING, regionLocation=a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:12,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731109749997, region=b47e8c6e0f01d522d8a759095cdb764d, REOPEN/MOVE because future has completed 2024-11-08T23:49:12,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b47e8c6e0f01d522d8a759095cdb764d, server=a15bf3ddec7f,33665,1731109747845}] 2024-11-08T23:49:12,327 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:12,327 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => b47e8c6e0f01d522d8a759095cdb764d, NAME => 'testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T23:49:12,327 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731109749997 b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,327 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:12,328 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,328 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,329 INFO [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,330 INFO [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b47e8c6e0f01d522d8a759095cdb764d columnFamilyName family 2024-11-08T23:49:12,330 DEBUG [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:12,332 DEBUG [StoreFileOpener-b47e8c6e0f01d522d8a759095cdb764d-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/family/f251d59e02064b3fbbe703acc71f7ef2, delay=987 ms 2024-11-08T23:49:12,333 DEBUG [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/family/f251d59e02064b3fbbe703acc71f7ef2 2024-11-08T23:49:12,334 INFO [StoreOpener-b47e8c6e0f01d522d8a759095cdb764d-1 {}] regionserver.HStore(327): Store=b47e8c6e0f01d522d8a759095cdb764d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T23:49:12,334 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,335 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,335 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,335 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,335 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,340 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,340 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened b47e8c6e0f01d522d8a759095cdb764d; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59892940, jitterRate=-0.10752564668655396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T23:49:12,340 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,341 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for b47e8c6e0f01d522d8a759095cdb764d: Running coprocessor pre-open hook at 1731109752328Writing region info on filesystem at 1731109752328Initializing all the Stores at 1731109752328Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109752328Cleaning up temporary data from old regions at 1731109752335 (+7 ms)Running coprocessor post-open hooks at 1731109752340 (+5 ms)Region opened successfully at 1731109752341 (+1 ms) 2024-11-08T23:49:12,342 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d., pid=9, masterSystemTime=1731109752324 2024-11-08T23:49:12,345 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:12,345 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:12,345 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b47e8c6e0f01d522d8a759095cdb764d, regionState=OPEN, openSeqNum=10, regionLocation=a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:12,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b47e8c6e0f01d522d8a759095cdb764d, server=a15bf3ddec7f,33665,1731109747845 because future has completed 2024-11-08T23:49:12,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-08T23:49:12,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure b47e8c6e0f01d522d8a759095cdb764d, server=a15bf3ddec7f,33665,1731109747845 in 177 msec 2024-11-08T23:49:12,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731109749997, region=b47e8c6e0f01d522d8a759095cdb764d, REOPEN/MOVE in 521 msec 2024-11-08T23:49:12,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T23:49:12,384 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T23:49:12,384 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T23:49:12,388 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:12,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:12,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T23:49:12,389 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T23:49:12,389 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1482236467, stopped=false 2024-11-08T23:49:12,389 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a15bf3ddec7f,41931,1731109747063 2024-11-08T23:49:12,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T23:49:12,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T23:49:12,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T23:49:12,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:12,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:12,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:12,392 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T23:49:12,393 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T23:49:12,393 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T23:49:12,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:12,393 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T23:49:12,393 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T23:49:12,393 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T23:49:12,393 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a15bf3ddec7f,33665,1731109747845' ***** 2024-11-08T23:49:12,393 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T23:49:12,393 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a15bf3ddec7f,44283,1731109747982' ***** 2024-11-08T23:49:12,393 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T23:49:12,394 INFO [RS:1;a15bf3ddec7f:44283 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T23:49:12,394 INFO [RS:0;a15bf3ddec7f:33665 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T23:49:12,394 INFO [RS:1;a15bf3ddec7f:44283 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/bucket.persistence 2024-11-08T23:49:12,394 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T23:49:12,394 INFO [RS:0;a15bf3ddec7f:33665 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/bucket.persistence 2024-11-08T23:49:12,394 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T23:49:12,394 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T23:49:12,394 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T23:49:12,394 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T23:49:12,394 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T23:49:12,395 INFO [RS:0;a15bf3ddec7f:33665 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T23:49:12,395 INFO [RS:1;a15bf3ddec7f:44283 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T23:49:12,395 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T23:49:12,395 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T23:49:12,405 DEBUG [RS:0;a15bf3ddec7f:33665 {}] bucket.BucketCache(1488): Thread RS:0;a15bf3ddec7f:33665 started persisting bucket cache to file 2024-11-08T23:49:12,405 DEBUG [RS:0;a15bf3ddec7f:33665 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T23:49:12,405 DEBUG [RS:1;a15bf3ddec7f:44283 {}] bucket.BucketCache(1488): Thread RS:1;a15bf3ddec7f:44283 started persisting bucket cache to file 2024-11-08T23:49:12,405 DEBUG [RS:0;a15bf3ddec7f:33665 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T23:49:12,405 DEBUG [RS:1;a15bf3ddec7f:44283 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T23:49:12,405 DEBUG [RS:1;a15bf3ddec7f:44283 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T23:49:12,409 DEBUG [RS:1;a15bf3ddec7f:44283 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T23:49:12,409 DEBUG [RS:1;a15bf3ddec7f:44283 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731109752405 2024-11-08T23:49:12,409 DEBUG [RS:0;a15bf3ddec7f:33665 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T23:49:12,409 DEBUG [RS:1;a15bf3ddec7f:44283 {}] bucket.BucketCache(1510): Thread RS:1;a15bf3ddec7f:44283 finished persisting bucket cache to file, renaming 2024-11-08T23:49:12,409 DEBUG [RS:0;a15bf3ddec7f:33665 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731109752405 2024-11-08T23:49:12,409 DEBUG [RS:0;a15bf3ddec7f:33665 {}] bucket.BucketCache(1510): Thread RS:0;a15bf3ddec7f:33665 finished persisting bucket cache to file, renaming 2024-11-08T23:49:12,409 WARN [RS:0;a15bf3ddec7f:33665 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-08T23:49:12,409 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T23:49:12,409 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T23:49:12,409 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T23:49:12,409 INFO [RS:1;a15bf3ddec7f:44283 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T23:49:12,409 INFO [RS:0;a15bf3ddec7f:33665 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T23:49:12,409 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T23:49:12,410 INFO [RS:1;a15bf3ddec7f:44283 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T23:49:12,410 INFO [RS:0;a15bf3ddec7f:33665 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T23:49:12,410 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(959): stopping server a15bf3ddec7f,44283,1731109747982 2024-11-08T23:49:12,410 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T23:49:12,410 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(3091): Received CLOSE for b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,410 INFO [RS:1;a15bf3ddec7f:44283 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a15bf3ddec7f:44283. 2024-11-08T23:49:12,410 DEBUG [RS:1;a15bf3ddec7f:44283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T23:49:12,410 DEBUG [RS:1;a15bf3ddec7f:44283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:12,410 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(976): stopping server a15bf3ddec7f,44283,1731109747982; all regions closed. 2024-11-08T23:49:12,411 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(959): stopping server a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:12,411 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T23:49:12,411 INFO [RS:0;a15bf3ddec7f:33665 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a15bf3ddec7f:33665. 2024-11-08T23:49:12,411 DEBUG [RS:0;a15bf3ddec7f:33665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T23:49:12,411 DEBUG [RS:0;a15bf3ddec7f:33665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:12,411 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T23:49:12,411 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T23:49:12,411 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T23:49:12,411 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T23:49:12,411 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T23:49:12,411 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b47e8c6e0f01d522d8a759095cdb764d, disabling compactions & flushes 2024-11-08T23:49:12,412 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:12,412 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-08T23:49:12,412 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:12,412 DEBUG [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(1325): Online Regions={b47e8c6e0f01d522d8a759095cdb764d=testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d., 1588230740=hbase:meta,,1.1588230740} 2024-11-08T23:49:12,412 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. after waiting 0 ms 2024-11-08T23:49:12,412 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T23:49:12,412 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:12,412 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T23:49:12,412 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T23:49:12,412 DEBUG [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, b47e8c6e0f01d522d8a759095cdb764d 2024-11-08T23:49:12,412 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T23:49:12,412 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T23:49:12,412 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T23:49:12,413 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-08T23:49:12,416 DEBUG [StoreFileCloser-b47e8c6e0f01d522d8a759095cdb764d-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/family/f251d59e02064b3fbbe703acc71f7ef2 2024-11-08T23:49:12,417 DEBUG [RS:1;a15bf3ddec7f:44283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs 2024-11-08T23:49:12,417 INFO [RS:1;a15bf3ddec7f:44283 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a15bf3ddec7f%2C44283%2C1731109747982:(num 1731109749363) 2024-11-08T23:49:12,417 DEBUG [RS:1;a15bf3ddec7f:44283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:12,417 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T23:49:12,417 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T23:49:12,417 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.ChoreService(370): Chore service for: regionserver/a15bf3ddec7f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T23:49:12,418 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T23:49:12,418 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T23:49:12,418 INFO [regionserver/a15bf3ddec7f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T23:49:12,418 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T23:49:12,418 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T23:49:12,418 INFO [RS:1;a15bf3ddec7f:44283 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44283 2024-11-08T23:49:12,422 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnRegionMove1731109749997/b47e8c6e0f01d522d8a759095cdb764d/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-08T23:49:12,422 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:12,422 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b47e8c6e0f01d522d8a759095cdb764d: Waiting for close lock at 1731109752411Running coprocessor pre-close hooks at 1731109752411Disabling compacts and flushes for region at 1731109752411Disabling writes for close at 1731109752412 (+1 ms)Writing region close event to WAL at 1731109752417 (+5 ms)Running coprocessor post-close hooks at 1731109752422 (+5 ms)Closed at 1731109752422 2024-11-08T23:49:12,422 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d. 2024-11-08T23:49:12,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a15bf3ddec7f,44283,1731109747982 2024-11-08T23:49:12,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T23:49:12,423 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T23:49:12,425 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 99a00cb8b4ba4dedbf764edbb6c86059 2024-11-08T23:49:12,426 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a15bf3ddec7f,44283,1731109747982] 2024-11-08T23:49:12,428 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a15bf3ddec7f,44283,1731109747982 already deleted, retry=false 2024-11-08T23:49:12,428 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a15bf3ddec7f,44283,1731109747982 expired; onlineServers=1 2024-11-08T23:49:12,442 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/info/99a00cb8b4ba4dedbf764edbb6c86059 is 205, key is testBlockEvictionOnRegionMove1731109749997,,1731109750017.b47e8c6e0f01d522d8a759095cdb764d./info:regioninfo/1731109752345/Put/seqid=0 2024-11-08T23:49:12,443 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/info/99a00cb8b4ba4dedbf764edbb6c86059 2024-11-08T23:49:12,454 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3b78602c4ccd481e88396153e2e1eb6d 2024-11-08T23:49:12,468 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/ns/3b78602c4ccd481e88396153e2e1eb6d is 43, key is default/ns:d/1731109749821/Put/seqid=0 2024-11-08T23:49:12,468 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/ns/3b78602c4ccd481e88396153e2e1eb6d 2024-11-08T23:49:12,477 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4a08fee6fa7d4157a220ef2529a3ea9c 2024-11-08T23:49:12,489 INFO [regionserver/a15bf3ddec7f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T23:49:12,489 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/table/4a08fee6fa7d4157a220ef2529a3ea9c is 78, key is testBlockEvictionOnRegionMove1731109749997/table:state/1731109750429/Put/seqid=0 2024-11-08T23:49:12,490 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/table/4a08fee6fa7d4157a220ef2529a3ea9c 2024-11-08T23:49:12,491 INFO [regionserver/a15bf3ddec7f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T23:49:12,493 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/info/99a00cb8b4ba4dedbf764edbb6c86059 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/99a00cb8b4ba4dedbf764edbb6c86059 2024-11-08T23:49:12,494 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/99a00cb8b4ba4dedbf764edbb6c86059, delay=970 ms 2024-11-08T23:49:12,495 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/99a00cb8b4ba4dedbf764edbb6c86059, entries=15, sequenceid=15, filesize=7.6 K 2024-11-08T23:49:12,496 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/ns/3b78602c4ccd481e88396153e2e1eb6d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/ns/3b78602c4ccd481e88396153e2e1eb6d 2024-11-08T23:49:12,497 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/ns/3b78602c4ccd481e88396153e2e1eb6d, delay=900 ms 2024-11-08T23:49:12,498 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/ns/3b78602c4ccd481e88396153e2e1eb6d, entries=2, sequenceid=15, filesize=5.0 K 2024-11-08T23:49:12,499 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/table/4a08fee6fa7d4157a220ef2529a3ea9c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/table/4a08fee6fa7d4157a220ef2529a3ea9c 2024-11-08T23:49:12,500 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/table/4a08fee6fa7d4157a220ef2529a3ea9c, delay=934 ms 2024-11-08T23:49:12,502 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/table/4a08fee6fa7d4157a220ef2529a3ea9c, entries=2, sequenceid=15, filesize=5.3 K 2024-11-08T23:49:12,503 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=15, compaction requested=false 2024-11-08T23:49:12,503 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T23:49:12,504 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/99a00cb8b4ba4dedbf764edbb6c86059 2024-11-08T23:49:12,505 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/ns/3b78602c4ccd481e88396153e2e1eb6d 2024-11-08T23:49:12,505 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/table/4a08fee6fa7d4157a220ef2529a3ea9c 2024-11-08T23:49:12,510 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-08T23:49:12,511 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T23:49:12,511 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T23:49:12,511 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731109752412Running coprocessor pre-close hooks at 1731109752412Disabling compacts and flushes for region at 1731109752412Disabling writes for close at 1731109752412Obtaining lock to block concurrent updates at 1731109752413 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731109752413Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731109752413Flushing stores of hbase:meta,,1.1588230740 at 1731109752413Flushing 1588230740/info: creating writer at 1731109752414 (+1 ms)Flushing 1588230740/info: appending metadata at 1731109752442 (+28 ms)Flushing 1588230740/info: closing flushed file at 1731109752442Flushing 1588230740/ns: creating writer at 1731109752446 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731109752467 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731109752467Flushing 1588230740/table: creating writer at 1731109752471 (+4 ms)Flushing 1588230740/table: appending metadata at 1731109752489 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731109752489Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dab5cc7: reopening flushed file at 1731109752493 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71eb1dae: reopening flushed file at 1731109752496 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@226f38f3: reopening flushed file at 1731109752499 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=15, compaction requested=false at 1731109752503 (+4 ms)Writing region close event to WAL at 1731109752506 (+3 ms)Running coprocessor post-close hooks at 1731109752510 (+4 ms)Closed at 1731109752511 (+1 ms) 2024-11-08T23:49:12,512 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T23:49:12,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:12,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44283-0x100f9fc62d50002, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:12,526 INFO [RS:1;a15bf3ddec7f:44283 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T23:49:12,526 INFO [RS:1;a15bf3ddec7f:44283 {}] regionserver.HRegionServer(1031): Exiting; stopping=a15bf3ddec7f,44283,1731109747982; zookeeper connection closed. 2024-11-08T23:49:12,527 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4ad36bfb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4ad36bfb 2024-11-08T23:49:12,612 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(976): stopping server a15bf3ddec7f,33665,1731109747845; all regions closed. 2024-11-08T23:49:12,615 DEBUG [RS:0;a15bf3ddec7f:33665 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs 2024-11-08T23:49:12,615 INFO [RS:0;a15bf3ddec7f:33665 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a15bf3ddec7f%2C33665%2C1731109747845.meta:.meta(num 1731109749694) 2024-11-08T23:49:12,617 DEBUG [RS:0;a15bf3ddec7f:33665 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs 2024-11-08T23:49:12,617 INFO [RS:0;a15bf3ddec7f:33665 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a15bf3ddec7f%2C33665%2C1731109747845:(num 1731109749363) 2024-11-08T23:49:12,617 DEBUG [RS:0;a15bf3ddec7f:33665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:12,617 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T23:49:12,617 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T23:49:12,617 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.ChoreService(370): Chore service for: regionserver/a15bf3ddec7f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T23:49:12,618 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T23:49:12,618 INFO [regionserver/a15bf3ddec7f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T23:49:12,618 INFO [RS:0;a15bf3ddec7f:33665 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33665 2024-11-08T23:49:12,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T23:49:12,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a15bf3ddec7f,33665,1731109747845 2024-11-08T23:49:12,620 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T23:49:12,621 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a15bf3ddec7f,33665,1731109747845] 2024-11-08T23:49:12,624 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a15bf3ddec7f,33665,1731109747845 already deleted, retry=false 2024-11-08T23:49:12,624 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a15bf3ddec7f,33665,1731109747845 expired; onlineServers=0 2024-11-08T23:49:12,625 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a15bf3ddec7f,41931,1731109747063' ***** 2024-11-08T23:49:12,625 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T23:49:12,625 INFO [M:0;a15bf3ddec7f:41931 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T23:49:12,625 INFO [M:0;a15bf3ddec7f:41931 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T23:49:12,625 DEBUG [M:0;a15bf3ddec7f:41931 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T23:49:12,625 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T23:49:12,625 DEBUG [M:0;a15bf3ddec7f:41931 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T23:49:12,625 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.large.0-1731109748967 {}] cleaner.HFileCleaner(306): Exit Thread[master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.large.0-1731109748967,5,FailOnTimeoutGroup] 2024-11-08T23:49:12,625 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.small.0-1731109748969 {}] cleaner.HFileCleaner(306): Exit Thread[master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.small.0-1731109748969,5,FailOnTimeoutGroup] 2024-11-08T23:49:12,625 INFO [M:0;a15bf3ddec7f:41931 {}] hbase.ChoreService(370): Chore service for: master/a15bf3ddec7f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T23:49:12,626 INFO [M:0;a15bf3ddec7f:41931 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T23:49:12,626 DEBUG [M:0;a15bf3ddec7f:41931 {}] master.HMaster(1795): Stopping service threads 2024-11-08T23:49:12,626 INFO [M:0;a15bf3ddec7f:41931 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T23:49:12,626 INFO [M:0;a15bf3ddec7f:41931 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T23:49:12,626 INFO [M:0;a15bf3ddec7f:41931 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T23:49:12,627 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T23:49:12,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T23:49:12,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:12,627 DEBUG [M:0;a15bf3ddec7f:41931 {}] zookeeper.ZKUtil(347): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T23:49:12,627 WARN [M:0;a15bf3ddec7f:41931 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T23:49:12,628 INFO [M:0;a15bf3ddec7f:41931 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/.lastflushedseqids 2024-11-08T23:49:12,641 INFO [M:0;a15bf3ddec7f:41931 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T23:49:12,641 INFO [M:0;a15bf3ddec7f:41931 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T23:49:12,641 DEBUG [M:0;a15bf3ddec7f:41931 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T23:49:12,641 INFO [M:0;a15bf3ddec7f:41931 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:12,641 DEBUG [M:0;a15bf3ddec7f:41931 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:12,641 DEBUG [M:0;a15bf3ddec7f:41931 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T23:49:12,641 DEBUG [M:0;a15bf3ddec7f:41931 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:12,641 INFO [M:0;a15bf3ddec7f:41931 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-08T23:49:12,651 DEBUG [M:0;a15bf3ddec7f:41931 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6af603b46366426399a3abb77ab87348 2024-11-08T23:49:12,665 DEBUG [M:0;a15bf3ddec7f:41931 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6af603b46366426399a3abb77ab87348 is 82, key is hbase:meta,,1/info:regioninfo/1731109749759/Put/seqid=0 2024-11-08T23:49:12,666 INFO [M:0;a15bf3ddec7f:41931 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6af603b46366426399a3abb77ab87348 2024-11-08T23:49:12,675 DEBUG [M:0;a15bf3ddec7f:41931 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a9ed8f3380544848b442a6d13ece0a70 2024-11-08T23:49:12,690 DEBUG [M:0;a15bf3ddec7f:41931 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a9ed8f3380544848b442a6d13ece0a70 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731109750435/Put/seqid=0 2024-11-08T23:49:12,691 INFO [M:0;a15bf3ddec7f:41931 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a9ed8f3380544848b442a6d13ece0a70 2024-11-08T23:49:12,700 DEBUG [M:0;a15bf3ddec7f:41931 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 44285773fc6441c596cbaa15f821e1a2 2024-11-08T23:49:12,713 DEBUG [M:0;a15bf3ddec7f:41931 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/44285773fc6441c596cbaa15f821e1a2 is 69, key is a15bf3ddec7f,33665,1731109747845/rs:state/1731109749106/Put/seqid=0 2024-11-08T23:49:12,714 INFO [M:0;a15bf3ddec7f:41931 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/44285773fc6441c596cbaa15f821e1a2 2024-11-08T23:49:12,716 DEBUG [M:0;a15bf3ddec7f:41931 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6af603b46366426399a3abb77ab87348 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6af603b46366426399a3abb77ab87348 2024-11-08T23:49:12,718 INFO [M:0;a15bf3ddec7f:41931 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6af603b46366426399a3abb77ab87348, entries=8, sequenceid=84, filesize=5.5 K 2024-11-08T23:49:12,718 DEBUG [M:0;a15bf3ddec7f:41931 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a9ed8f3380544848b442a6d13ece0a70 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a9ed8f3380544848b442a6d13ece0a70 2024-11-08T23:49:12,720 INFO [M:0;a15bf3ddec7f:41931 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a9ed8f3380544848b442a6d13ece0a70, entries=9, sequenceid=84, filesize=6.5 K 2024-11-08T23:49:12,721 DEBUG [M:0;a15bf3ddec7f:41931 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/44285773fc6441c596cbaa15f821e1a2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/44285773fc6441c596cbaa15f821e1a2 2024-11-08T23:49:12,723 INFO [M:0;a15bf3ddec7f:41931 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/44285773fc6441c596cbaa15f821e1a2, entries=2, sequenceid=84, filesize=5.1 K 2024-11-08T23:49:12,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:12,724 INFO [RS:0;a15bf3ddec7f:33665 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T23:49:12,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33665-0x100f9fc62d50001, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:12,724 INFO [M:0;a15bf3ddec7f:41931 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=84, compaction requested=false 2024-11-08T23:49:12,724 INFO [RS:0;a15bf3ddec7f:33665 {}] regionserver.HRegionServer(1031): Exiting; stopping=a15bf3ddec7f,33665,1731109747845; zookeeper connection closed. 2024-11-08T23:49:12,724 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c0a11d4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c0a11d4 2024-11-08T23:49:12,724 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-08T23:49:12,727 INFO [M:0;a15bf3ddec7f:41931 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:12,727 DEBUG [M:0;a15bf3ddec7f:41931 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731109752641Disabling compacts and flushes for region at 1731109752641Disabling writes for close at 1731109752641Obtaining lock to block concurrent updates at 1731109752641Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731109752641Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731109752642 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731109752642Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731109752642Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731109752665 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731109752665Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731109752668 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731109752689 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731109752689Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731109752694 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731109752713 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731109752713Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e1d7f5e: reopening flushed file at 1731109752716 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e12b836: reopening flushed file at 1731109752718 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48e80c52: reopening flushed file at 1731109752720 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=84, compaction requested=false at 1731109752724 (+4 ms)Writing region close event to WAL at 1731109752727 (+3 ms)Closed at 1731109752727 2024-11-08T23:49:12,729 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T23:49:12,729 INFO [M:0;a15bf3ddec7f:41931 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T23:49:12,729 INFO [M:0;a15bf3ddec7f:41931 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41931 2024-11-08T23:49:12,730 INFO [M:0;a15bf3ddec7f:41931 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T23:49:12,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:12,832 INFO [M:0;a15bf3ddec7f:41931 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T23:49:12,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41931-0x100f9fc62d50000, quorum=127.0.0.1:60153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:12,837 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T23:49:12,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T23:49:12,857 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:12,865 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/a15bf3ddec7f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/a15bf3ddec7f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731109752332 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731109752497 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/a15bf3ddec7f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/a15bf3ddec7f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1731109752494 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731109750808 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1606 (was 1871), ProcessCount=11 (was 11), AvailableMemoryMB=3318 (was 3666) 2024-11-08T23:49:12,871 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=1606, ProcessCount=11, AvailableMemoryMB=3318 2024-11-08T23:49:12,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/hadoop-log-dir so I do NOT create it in target/test-data/27608b39-682c-5882-6349-9cffd4fd803b 2024-11-08T23:49:12,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/hadoop-tmp-dir so I do NOT create it in target/test-data/27608b39-682c-5882-6349-9cffd4fd803b 2024-11-08T23:49:12,872 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:12,875 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/cluster_ad600924-2be5-8ced-66f8-3f0723cf198d, deleteOnExit=true 2024-11-08T23:49:12,877 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/cluster_ad600924-2be5-8ced-66f8-3f0723cf198d/zookeeper_0, clientPort=49539, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/cluster_ad600924-2be5-8ced-66f8-3f0723cf198d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/cluster_ad600924-2be5-8ced-66f8-3f0723cf198d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T23:49:12,879 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49539 2024-11-08T23:49:12,890 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59 with version=8 2024-11-08T23:49:12,890 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/hbase-staging 2024-11-08T23:49:12,893 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a15bf3ddec7f:0 server-side Connection retries=45 2024-11-08T23:49:12,893 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:12,894 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:12,894 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T23:49:12,894 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:12,894 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T23:49:12,894 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T23:49:12,894 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T23:49:12,895 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46357 2024-11-08T23:49:12,896 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46357 connecting to ZooKeeper ensemble=127.0.0.1:49539 2024-11-08T23:49:12,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:463570x0, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T23:49:12,903 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46357-0x100f9fc7abc0000 connected 2024-11-08T23:49:12,919 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:12,919 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:12,920 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T23:49:12,920 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59, hbase.cluster.distributed=false 2024-11-08T23:49:12,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T23:49:12,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46357 2024-11-08T23:49:12,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46357 2024-11-08T23:49:12,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46357 2024-11-08T23:49:12,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46357 2024-11-08T23:49:12,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46357 2024-11-08T23:49:12,940 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a15bf3ddec7f:0 server-side Connection retries=45 2024-11-08T23:49:12,940 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:12,941 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:12,941 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T23:49:12,941 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:12,941 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T23:49:12,941 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T23:49:12,941 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T23:49:12,942 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46179 2024-11-08T23:49:12,943 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46179 connecting to ZooKeeper ensemble=127.0.0.1:49539 2024-11-08T23:49:12,943 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:12,944 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:12,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461790x0, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T23:49:12,947 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46179-0x100f9fc7abc0001 connected 2024-11-08T23:49:12,947 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T23:49:12,948 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T23:49:12,949 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.cache 2024-11-08T23:49:12,949 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T23:49:12,949 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T23:49:12,949 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T23:49:12,952 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T23:49:12,952 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T23:49:12,953 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T23:49:12,953 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T23:49:12,953 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.persistence is complete. 2024-11-08T23:49:12,953 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T23:49:12,954 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T23:49:12,955 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T23:49:12,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46179 2024-11-08T23:49:12,959 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46179 2024-11-08T23:49:12,960 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46179 2024-11-08T23:49:12,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46179 2024-11-08T23:49:12,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46179 2024-11-08T23:49:12,979 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a15bf3ddec7f:0 server-side Connection retries=45 2024-11-08T23:49:12,979 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:12,979 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:12,979 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T23:49:12,979 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:12,979 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T23:49:12,979 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T23:49:12,980 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T23:49:12,980 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45685 2024-11-08T23:49:12,982 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45685 connecting to ZooKeeper ensemble=127.0.0.1:49539 2024-11-08T23:49:12,982 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:12,982 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:12,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456850x0, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T23:49:12,985 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45685-0x100f9fc7abc0002 connected 2024-11-08T23:49:12,985 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T23:49:12,986 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T23:49:12,986 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.cache 2024-11-08T23:49:12,987 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T23:49:12,987 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T23:49:12,987 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T23:49:12,987 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T23:49:12,987 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T23:49:12,987 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T23:49:12,988 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T23:49:12,988 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.persistence is complete. 2024-11-08T23:49:12,988 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T23:49:12,989 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T23:49:12,990 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T23:49:12,990 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45685 2024-11-08T23:49:12,990 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45685 2024-11-08T23:49:12,991 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45685 2024-11-08T23:49:12,991 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45685 2024-11-08T23:49:12,991 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45685 2024-11-08T23:49:13,011 DEBUG [M:0;a15bf3ddec7f:46357 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a15bf3ddec7f:46357 2024-11-08T23:49:13,011 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a15bf3ddec7f,46357,1731109752893 2024-11-08T23:49:13,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T23:49:13,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T23:49:13,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T23:49:13,014 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a15bf3ddec7f,46357,1731109752893 2024-11-08T23:49:13,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T23:49:13,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T23:49:13,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,019 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T23:49:13,019 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a15bf3ddec7f,46357,1731109752893 from backup master directory 2024-11-08T23:49:13,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T23:49:13,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T23:49:13,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a15bf3ddec7f,46357,1731109752893 2024-11-08T23:49:13,021 WARN [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T23:49:13,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T23:49:13,021 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a15bf3ddec7f,46357,1731109752893 2024-11-08T23:49:13,023 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/hbase.id] with ID: ced4ba26-b3e7-47b2-ab2d-bf8c5ffc4f58 2024-11-08T23:49:13,023 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/.tmp/hbase.id 2024-11-08T23:49:13,028 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/hbase.id] 2024-11-08T23:49:13,069 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:13,069 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T23:49:13,072 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-08T23:49:13,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,079 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T23:49:13,080 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T23:49:13,081 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T23:49:13,101 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store 2024-11-08T23:49:13,107 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:13,108 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T23:49:13,108 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:13,108 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:13,108 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T23:49:13,108 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:13,108 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:13,108 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731109753108Disabling compacts and flushes for region at 1731109753108Disabling writes for close at 1731109753108Writing region close event to WAL at 1731109753108Closed at 1731109753108 2024-11-08T23:49:13,108 WARN [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/.initializing 2024-11-08T23:49:13,108 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/WALs/a15bf3ddec7f,46357,1731109752893 2024-11-08T23:49:13,114 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T23:49:13,119 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a15bf3ddec7f%2C46357%2C1731109752893, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/WALs/a15bf3ddec7f,46357,1731109752893, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/oldWALs, maxLogs=10 2024-11-08T23:49:13,136 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/WALs/a15bf3ddec7f,46357,1731109752893/a15bf3ddec7f%2C46357%2C1731109752893.1731109753119 2024-11-08T23:49:13,137 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T23:49:13,137 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T23:49:13,137 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:13,137 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:13,137 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:13,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:13,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T23:49:13,143 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:13,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:13,147 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T23:49:13,147 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,147 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T23:49:13,147 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:13,151 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T23:49:13,151 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T23:49:13,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:13,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T23:49:13,156 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,156 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T23:49:13,156 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:13,156 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:13,156 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:13,157 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:13,157 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:13,157 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T23:49:13,157 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T23:49:13,164 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T23:49:13,164 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59197575, jitterRate=-0.11788739264011383}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T23:49:13,165 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731109753138Initializing all the Stores at 1731109753138Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109753138Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109753138Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109753138Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109753138Cleaning up temporary data from old regions at 1731109753157 (+19 ms)Region opened successfully at 1731109753165 (+8 ms) 2024-11-08T23:49:13,165 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T23:49:13,169 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cfa53ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a15bf3ddec7f/172.17.0.2:0 2024-11-08T23:49:13,170 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T23:49:13,171 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T23:49:13,171 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T23:49:13,171 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T23:49:13,171 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T23:49:13,171 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-08T23:49:13,172 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T23:49:13,174 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T23:49:13,175 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T23:49:13,176 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T23:49:13,177 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T23:49:13,177 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T23:49:13,179 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T23:49:13,179 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T23:49:13,180 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T23:49:13,182 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T23:49:13,183 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T23:49:13,184 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T23:49:13,186 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T23:49:13,187 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T23:49:13,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T23:49:13,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T23:49:13,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T23:49:13,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,190 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a15bf3ddec7f,46357,1731109752893, sessionid=0x100f9fc7abc0000, setting cluster-up flag (Was=false) 2024-11-08T23:49:13,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,200 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T23:49:13,202 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a15bf3ddec7f,46357,1731109752893 2024-11-08T23:49:13,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,210 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T23:49:13,211 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a15bf3ddec7f,46357,1731109752893 2024-11-08T23:49:13,212 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T23:49:13,214 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T23:49:13,214 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T23:49:13,215 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T23:49:13,215 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a15bf3ddec7f,46357,1731109752893 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T23:49:13,217 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a15bf3ddec7f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T23:49:13,217 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a15bf3ddec7f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T23:49:13,217 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T23:49:13,217 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T23:49:13,217 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a15bf3ddec7f:0, corePoolSize=10, maxPoolSize=10 2024-11-08T23:49:13,217 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,218 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T23:49:13,218 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,219 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109783218 2024-11-08T23:49:13,219 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T23:49:13,219 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T23:49:13,219 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T23:49:13,219 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T23:49:13,219 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T23:49:13,219 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T23:49:13,219 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T23:49:13,219 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T23:49:13,220 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,220 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T23:49:13,220 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,221 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T23:49:13,221 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T23:49:13,221 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T23:49:13,221 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T23:49:13,221 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T23:49:13,224 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.large.0-1731109753221,5,FailOnTimeoutGroup] 2024-11-08T23:49:13,224 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.small.0-1731109753224,5,FailOnTimeoutGroup] 2024-11-08T23:49:13,224 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,224 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T23:49:13,224 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,224 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,237 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T23:49:13,237 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59 2024-11-08T23:49:13,245 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:13,246 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T23:49:13,250 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T23:49:13,250 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:13,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T23:49:13,254 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T23:49:13,254 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:13,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T23:49:13,258 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T23:49:13,258 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,258 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:13,258 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T23:49:13,261 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T23:49:13,261 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,262 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:13,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T23:49:13,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740 2024-11-08T23:49:13,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740 2024-11-08T23:49:13,263 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T23:49:13,263 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T23:49:13,263 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T23:49:13,263 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T23:49:13,269 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T23:49:13,269 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65082487, jitterRate=-0.030195370316505432}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T23:49:13,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731109753245Initializing all the Stores at 1731109753245Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109753245Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109753246 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109753246Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109753246Cleaning up temporary data from old regions at 1731109753263 (+17 ms)Region opened successfully at 1731109753270 (+7 ms) 2024-11-08T23:49:13,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T23:49:13,271 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T23:49:13,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T23:49:13,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T23:49:13,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T23:49:13,271 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T23:49:13,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731109753270Disabling compacts and flushes for region at 1731109753270Disabling writes for close at 1731109753271 (+1 ms)Writing region close event to WAL at 1731109753271Closed at 1731109753271 2024-11-08T23:49:13,272 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T23:49:13,272 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T23:49:13,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T23:49:13,273 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T23:49:13,274 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T23:49:13,294 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(746): ClusterId : ced4ba26-b3e7-47b2-ab2d-bf8c5ffc4f58 2024-11-08T23:49:13,294 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(746): ClusterId : ced4ba26-b3e7-47b2-ab2d-bf8c5ffc4f58 2024-11-08T23:49:13,294 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T23:49:13,294 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T23:49:13,296 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T23:49:13,297 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T23:49:13,297 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T23:49:13,297 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T23:49:13,301 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T23:49:13,301 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T23:49:13,301 DEBUG [RS:1;a15bf3ddec7f:45685 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b370246, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a15bf3ddec7f/172.17.0.2:0 2024-11-08T23:49:13,302 DEBUG [RS:0;a15bf3ddec7f:46179 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284937c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a15bf3ddec7f/172.17.0.2:0 2024-11-08T23:49:13,314 DEBUG [RS:0;a15bf3ddec7f:46179 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a15bf3ddec7f:46179 2024-11-08T23:49:13,314 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T23:49:13,314 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T23:49:13,314 DEBUG [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T23:49:13,315 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(2659): reportForDuty to master=a15bf3ddec7f,46357,1731109752893 with port=46179, startcode=1731109752940 2024-11-08T23:49:13,315 DEBUG [RS:1;a15bf3ddec7f:45685 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a15bf3ddec7f:45685 2024-11-08T23:49:13,316 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T23:49:13,316 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T23:49:13,316 DEBUG [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T23:49:13,316 DEBUG [RS:0;a15bf3ddec7f:46179 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T23:49:13,316 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(2659): reportForDuty to master=a15bf3ddec7f,46357,1731109752893 with port=45685, startcode=1731109752978 2024-11-08T23:49:13,317 DEBUG [RS:1;a15bf3ddec7f:45685 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T23:49:13,318 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42779, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T23:49:13,319 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42315, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T23:49:13,319 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a15bf3ddec7f,46179,1731109752940 2024-11-08T23:49:13,319 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46357 {}] master.ServerManager(517): Registering regionserver=a15bf3ddec7f,46179,1731109752940 2024-11-08T23:49:13,321 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:13,321 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46357 {}] master.ServerManager(517): Registering regionserver=a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:13,321 DEBUG [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59 2024-11-08T23:49:13,321 DEBUG [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T23:49:13,321 DEBUG [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T23:49:13,322 DEBUG [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59 2024-11-08T23:49:13,322 DEBUG [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T23:49:13,322 DEBUG [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T23:49:13,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T23:49:13,328 DEBUG [RS:0;a15bf3ddec7f:46179 {}] zookeeper.ZKUtil(111): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a15bf3ddec7f,46179,1731109752940 2024-11-08T23:49:13,328 WARN [RS:0;a15bf3ddec7f:46179 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T23:49:13,328 INFO [RS:0;a15bf3ddec7f:46179 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T23:49:13,328 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a15bf3ddec7f,45685,1731109752978] 2024-11-08T23:49:13,329 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a15bf3ddec7f,46179,1731109752940] 2024-11-08T23:49:13,329 DEBUG [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,46179,1731109752940 2024-11-08T23:49:13,329 DEBUG [RS:1;a15bf3ddec7f:45685 {}] zookeeper.ZKUtil(111): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:13,329 WARN [RS:1;a15bf3ddec7f:45685 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T23:49:13,329 INFO [RS:1;a15bf3ddec7f:45685 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T23:49:13,329 DEBUG [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:13,335 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T23:49:13,337 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T23:49:13,337 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T23:49:13,337 INFO [RS:0;a15bf3ddec7f:46179 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T23:49:13,337 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,337 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T23:49:13,338 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T23:49:13,338 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,338 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,338 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,338 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T23:49:13,338 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,338 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,338 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,338 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a15bf3ddec7f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T23:49:13,339 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,339 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,339 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,339 INFO [RS:1;a15bf3ddec7f:45685 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T23:49:13,339 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,339 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,339 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,339 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,339 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a15bf3ddec7f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T23:49:13,339 DEBUG [RS:0;a15bf3ddec7f:46179 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a15bf3ddec7f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T23:49:13,339 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T23:49:13,339 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,339 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T23:49:13,339 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,340 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,340 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,340 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,340 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,340 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46179,1731109752940-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T23:49:13,340 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,340 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,340 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,340 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,340 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,340 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a15bf3ddec7f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T23:49:13,340 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,340 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,340 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,341 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,341 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,341 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:13,341 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a15bf3ddec7f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T23:49:13,341 DEBUG [RS:1;a15bf3ddec7f:45685 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a15bf3ddec7f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T23:49:13,344 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,344 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,345 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,345 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,345 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,345 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,45685,1731109752978-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T23:49:13,355 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T23:49:13,356 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46179,1731109752940-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,356 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,356 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.Replication(171): a15bf3ddec7f,46179,1731109752940 started 2024-11-08T23:49:13,366 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T23:49:13,367 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,45685,1731109752978-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,367 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,367 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.Replication(171): a15bf3ddec7f,45685,1731109752978 started 2024-11-08T23:49:13,370 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,371 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(1482): Serving as a15bf3ddec7f,46179,1731109752940, RpcServer on a15bf3ddec7f/172.17.0.2:46179, sessionid=0x100f9fc7abc0001 2024-11-08T23:49:13,371 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T23:49:13,371 DEBUG [RS:0;a15bf3ddec7f:46179 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a15bf3ddec7f,46179,1731109752940 2024-11-08T23:49:13,371 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a15bf3ddec7f,46179,1731109752940' 2024-11-08T23:49:13,371 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T23:49:13,372 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T23:49:13,372 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T23:49:13,372 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T23:49:13,372 DEBUG [RS:0;a15bf3ddec7f:46179 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a15bf3ddec7f,46179,1731109752940 2024-11-08T23:49:13,372 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a15bf3ddec7f,46179,1731109752940' 2024-11-08T23:49:13,373 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T23:49:13,373 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T23:49:13,374 DEBUG [RS:0;a15bf3ddec7f:46179 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T23:49:13,374 INFO [RS:0;a15bf3ddec7f:46179 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T23:49:13,374 INFO [RS:0;a15bf3ddec7f:46179 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T23:49:13,382 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,382 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(1482): Serving as a15bf3ddec7f,45685,1731109752978, RpcServer on a15bf3ddec7f/172.17.0.2:45685, sessionid=0x100f9fc7abc0002 2024-11-08T23:49:13,382 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T23:49:13,382 DEBUG [RS:1;a15bf3ddec7f:45685 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:13,382 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a15bf3ddec7f,45685,1731109752978' 2024-11-08T23:49:13,382 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T23:49:13,383 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T23:49:13,383 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T23:49:13,383 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T23:49:13,383 DEBUG [RS:1;a15bf3ddec7f:45685 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:13,383 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a15bf3ddec7f,45685,1731109752978' 2024-11-08T23:49:13,383 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T23:49:13,384 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T23:49:13,384 DEBUG [RS:1;a15bf3ddec7f:45685 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T23:49:13,384 INFO [RS:1;a15bf3ddec7f:45685 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T23:49:13,384 INFO [RS:1;a15bf3ddec7f:45685 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T23:49:13,424 WARN [a15bf3ddec7f:46357 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T23:49:13,474 INFO [RS:0;a15bf3ddec7f:46179 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T23:49:13,475 INFO [RS:0;a15bf3ddec7f:46179 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a15bf3ddec7f%2C46179%2C1731109752940, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,46179,1731109752940, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs, maxLogs=55 2024-11-08T23:49:13,485 INFO [RS:1;a15bf3ddec7f:45685 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T23:49:13,485 INFO [RS:1;a15bf3ddec7f:45685 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a15bf3ddec7f%2C45685%2C1731109752978, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,45685,1731109752978, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs, maxLogs=55 2024-11-08T23:49:13,494 INFO [RS:0;a15bf3ddec7f:46179 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,46179,1731109752940/a15bf3ddec7f%2C46179%2C1731109752940.1731109753477 2024-11-08T23:49:13,495 DEBUG [RS:0;a15bf3ddec7f:46179 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T23:49:13,504 INFO [RS:1;a15bf3ddec7f:45685 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,45685,1731109752978/a15bf3ddec7f%2C45685%2C1731109752978.1731109753487 2024-11-08T23:49:13,504 DEBUG [RS:1;a15bf3ddec7f:45685 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T23:49:13,675 DEBUG [a15bf3ddec7f:46357 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T23:49:13,675 DEBUG [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(204): Hosts are {a15bf3ddec7f=0} racks are {/default-rack=0} 2024-11-08T23:49:13,676 DEBUG [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T23:49:13,676 DEBUG [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T23:49:13,676 DEBUG [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T23:49:13,676 DEBUG [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T23:49:13,676 INFO [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T23:49:13,676 INFO [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T23:49:13,676 DEBUG [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T23:49:13,677 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:13,678 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,45685,1731109752978, state=OPENING 2024-11-08T23:49:13,679 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T23:49:13,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:13,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:13,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:13,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:13,682 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T23:49:13,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a15bf3ddec7f,45685,1731109752978}] 2024-11-08T23:49:13,835 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T23:49:13,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58575, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T23:49:13,840 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T23:49:13,840 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T23:49:13,840 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T23:49:13,841 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a15bf3ddec7f%2C45685%2C1731109752978.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,45685,1731109752978, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs, maxLogs=55 2024-11-08T23:49:13,860 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,45685,1731109752978/a15bf3ddec7f%2C45685%2C1731109752978.meta.1731109753841.meta 2024-11-08T23:49:13,860 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T23:49:13,860 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T23:49:13,861 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T23:49:13,861 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T23:49:13,861 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T23:49:13,861 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T23:49:13,861 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:13,861 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T23:49:13,862 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T23:49:13,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T23:49:13,863 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T23:49:13,863 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:13,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T23:49:13,864 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T23:49:13,864 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:13,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T23:49:13,865 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T23:49:13,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:13,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T23:49:13,866 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T23:49:13,866 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:13,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:13,866 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T23:49:13,867 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740 2024-11-08T23:49:13,868 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740 2024-11-08T23:49:13,868 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T23:49:13,868 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T23:49:13,868 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T23:49:13,869 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T23:49:13,869 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66463444, jitterRate=-0.009617507457733154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T23:49:13,869 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T23:49:13,870 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731109753862Writing region info on filesystem at 1731109753862Initializing all the Stores at 1731109753862Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109753862Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109753862Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109753862Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109753862Cleaning up temporary data from old regions at 1731109753868 (+6 ms)Running coprocessor post-open hooks at 1731109753870 (+2 ms)Region opened successfully at 1731109753870 2024-11-08T23:49:13,871 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731109753835 2024-11-08T23:49:13,874 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T23:49:13,874 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T23:49:13,874 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:13,875 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,45685,1731109752978, state=OPEN 2024-11-08T23:49:13,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T23:49:13,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T23:49:13,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T23:49:13,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:13,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:13,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:13,878 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:13,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T23:49:13,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a15bf3ddec7f,45685,1731109752978 in 196 msec 2024-11-08T23:49:13,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T23:49:13,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec 2024-11-08T23:49:13,883 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T23:49:13,883 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T23:49:13,887 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T23:49:13,887 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a15bf3ddec7f,45685,1731109752978, seqNum=-1] 2024-11-08T23:49:13,887 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T23:49:13,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46873, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T23:49:13,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 680 msec 2024-11-08T23:49:13,894 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731109753894, completionTime=-1 2024-11-08T23:49:13,894 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-08T23:49:13,894 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T23:49:13,897 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T23:49:13,897 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109813897 2024-11-08T23:49:13,897 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109873897 2024-11-08T23:49:13,897 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-08T23:49:13,897 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46357,1731109752893-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,897 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46357,1731109752893-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,897 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46357,1731109752893-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,897 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a15bf3ddec7f:46357, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,898 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,898 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,900 DEBUG [master/a15bf3ddec7f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T23:49:13,902 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.881sec 2024-11-08T23:49:13,903 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T23:49:13,903 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T23:49:13,903 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T23:49:13,903 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T23:49:13,903 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T23:49:13,903 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46357,1731109752893-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T23:49:13,903 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46357,1731109752893-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T23:49:13,904 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T23:49:13,904 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T23:49:13,904 INFO [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46357,1731109752893-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:13,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T23:49:13,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a15bf3ddec7f,46357,-1 for getting cluster id 2024-11-08T23:49:13,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T23:49:13,997 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ced4ba26-b3e7-47b2-ab2d-bf8c5ffc4f58' 2024-11-08T23:49:13,997 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T23:49:13,997 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ced4ba26-b3e7-47b2-ab2d-bf8c5ffc4f58" 2024-11-08T23:49:13,998 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38048e75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T23:49:13,998 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a15bf3ddec7f,46357,-1] 2024-11-08T23:49:13,998 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T23:49:13,998 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:14,000 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T23:49:14,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af0bbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T23:49:14,001 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T23:49:14,003 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a15bf3ddec7f,45685,1731109752978, seqNum=-1] 2024-11-08T23:49:14,003 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T23:49:14,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T23:49:14,007 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a15bf3ddec7f,46357,1731109752893 2024-11-08T23:49:14,007 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T23:49:14,008 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is a15bf3ddec7f,46357,1731109752893 2024-11-08T23:49:14,008 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4ed573f 2024-11-08T23:49:14,009 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T23:49:14,010 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T23:49:14,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46357 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731109754007', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T23:49:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46357 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731109754007 2024-11-08T23:49:14,013 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731109754007 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T23:49:14,013 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:14,013 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46357 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731109754007" procId is: 4 2024-11-08T23:49:14,014 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731109754007 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T23:49:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T23:49:14,025 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731109754007-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 450433d2ce3191256fcef97bd2b0549e, NAME => 'testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731109754007', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59 2024-11-08T23:49:14,033 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731109754007-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:14,033 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731109754007-pool-0 {}] regionserver.HRegion(1722): Closing 450433d2ce3191256fcef97bd2b0549e, disabling compactions & flushes 2024-11-08T23:49:14,033 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731109754007-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:14,033 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731109754007-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:14,033 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731109754007-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. after waiting 0 ms 2024-11-08T23:49:14,033 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731109754007-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:14,033 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731109754007-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:14,033 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731109754007-pool-0 {}] regionserver.HRegion(1676): Region close journal for 450433d2ce3191256fcef97bd2b0549e: Waiting for close lock at 1731109754033Disabling compacts and flushes for region at 1731109754033Disabling writes for close at 1731109754033Writing region close event to WAL at 1731109754033Closed at 1731109754033 2024-11-08T23:49:14,034 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731109754007 execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T23:49:14,035 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731109754034"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731109754034"}]},"ts":"1731109754034"} 2024-11-08T23:49:14,037 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T23:49:14,038 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731109754007 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T23:49:14,038 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731109754007","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109754038"}]},"ts":"1731109754038"} 2024-11-08T23:49:14,040 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731109754007, state=ENABLING in hbase:meta 2024-11-08T23:49:14,040 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a15bf3ddec7f=0} racks are {/default-rack=0} 2024-11-08T23:49:14,041 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T23:49:14,041 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T23:49:14,041 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T23:49:14,041 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T23:49:14,041 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T23:49:14,041 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T23:49:14,041 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T23:49:14,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731109754007, region=450433d2ce3191256fcef97bd2b0549e, ASSIGN}] 2024-11-08T23:49:14,042 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731109754007, region=450433d2ce3191256fcef97bd2b0549e, ASSIGN 2024-11-08T23:49:14,043 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731109754007, region=450433d2ce3191256fcef97bd2b0549e, ASSIGN; state=OFFLINE, location=a15bf3ddec7f,45685,1731109752978; forceNewPlan=false, retain=false 2024-11-08T23:49:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T23:49:14,194 INFO [a15bf3ddec7f:46357 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T23:49:14,194 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=450433d2ce3191256fcef97bd2b0549e, regionState=OPENING, regionLocation=a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:14,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731109754007, region=450433d2ce3191256fcef97bd2b0549e, ASSIGN because future has completed 2024-11-08T23:49:14,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 450433d2ce3191256fcef97bd2b0549e, server=a15bf3ddec7f,45685,1731109752978}] 2024-11-08T23:49:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T23:49:14,352 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:14,353 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 450433d2ce3191256fcef97bd2b0549e, NAME => 'testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e.', STARTKEY => '', ENDKEY => ''} 2024-11-08T23:49:14,353 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731109754007 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:14,353 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:14,353 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:14,353 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:14,354 INFO [StoreOpener-450433d2ce3191256fcef97bd2b0549e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:14,358 INFO [StoreOpener-450433d2ce3191256fcef97bd2b0549e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 450433d2ce3191256fcef97bd2b0549e columnFamilyName family 2024-11-08T23:49:14,358 DEBUG [StoreOpener-450433d2ce3191256fcef97bd2b0549e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:14,358 INFO [StoreOpener-450433d2ce3191256fcef97bd2b0549e-1 {}] regionserver.HStore(327): Store=450433d2ce3191256fcef97bd2b0549e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T23:49:14,358 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:14,359 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:14,359 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:14,359 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:14,359 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:14,360 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:14,366 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T23:49:14,366 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 450433d2ce3191256fcef97bd2b0549e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69607000, jitterRate=0.03722512722015381}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T23:49:14,366 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:14,367 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 450433d2ce3191256fcef97bd2b0549e: Running coprocessor pre-open hook at 1731109754353Writing region info on filesystem at 1731109754353Initializing all the Stores at 1731109754354 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109754354Cleaning up temporary data from old regions at 1731109754359 (+5 ms)Running coprocessor post-open hooks at 1731109754366 (+7 ms)Region opened successfully at 1731109754367 (+1 ms) 2024-11-08T23:49:14,368 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e., pid=6, masterSystemTime=1731109754350 2024-11-08T23:49:14,370 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:14,370 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:14,371 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=450433d2ce3191256fcef97bd2b0549e, regionState=OPEN, openSeqNum=2, regionLocation=a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:14,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 450433d2ce3191256fcef97bd2b0549e, server=a15bf3ddec7f,45685,1731109752978 because future has completed 2024-11-08T23:49:14,375 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T23:49:14,375 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 450433d2ce3191256fcef97bd2b0549e, server=a15bf3ddec7f,45685,1731109752978 in 176 msec 2024-11-08T23:49:14,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T23:49:14,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731109754007, region=450433d2ce3191256fcef97bd2b0549e, ASSIGN in 334 msec 2024-11-08T23:49:14,379 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731109754007 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T23:49:14,379 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731109754007","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109754379"}]},"ts":"1731109754379"} 2024-11-08T23:49:14,381 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731109754007, state=ENABLED in hbase:meta 2024-11-08T23:49:14,381 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731109754007 execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T23:49:14,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731109754007 in 370 msec 2024-11-08T23:49:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T23:49:14,642 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731109754007 get assigned. Timeout = 60000ms 2024-11-08T23:49:14,642 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731109754007 completed 2024-11-08T23:49:14,642 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T23:49:14,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731109754007 assigned to meta. Checking AM states. 2024-11-08T23:49:14,646 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T23:49:14,646 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731109754007 assigned. 2024-11-08T23:49:14,649 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731109754007', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e., hostname=a15bf3ddec7f,45685,1731109752978, seqNum=2] 2024-11-08T23:49:14,654 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 450433d2ce3191256fcef97bd2b0549e 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-08T23:49:14,664 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on aa1464c143d04cb1ac99f03029035a60 2024-11-08T23:49:14,678 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/.tmp/family/aa1464c143d04cb1ac99f03029035a60 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-08T23:49:14,679 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/.tmp/family/aa1464c143d04cb1ac99f03029035a60 2024-11-08T23:49:14,682 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/.tmp/family/aa1464c143d04cb1ac99f03029035a60 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/family/aa1464c143d04cb1ac99f03029035a60 2024-11-08T23:49:14,683 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/family/aa1464c143d04cb1ac99f03029035a60, delay=928 ms 2024-11-08T23:49:14,684 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/family/aa1464c143d04cb1ac99f03029035a60, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T23:49:14,684 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 450433d2ce3191256fcef97bd2b0549e in 30ms, sequenceid=6, compaction requested=false 2024-11-08T23:49:14,685 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731109754007' 2024-11-08T23:49:14,685 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 450433d2ce3191256fcef97bd2b0549e: 2024-11-08T23:49:15,231 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T23:49:15,611 DEBUG [hfile-prefetch-1731109750808 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key aa1464c143d04cb1ac99f03029035a60_0. 2024-11-08T23:49:15,612 DEBUG [hfile-prefetch-1731109750808 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key aa1464c143d04cb1ac99f03029035a60_117. 2024-11-08T23:49:15,612 DEBUG [hfile-prefetch-1731109750808 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/family/aa1464c143d04cb1ac99f03029035a60, with total blocks 2, and data blocks 1 2024-11-08T23:49:15,613 DEBUG [hfile-prefetch-1731109750808 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-08T23:49:15,613 DEBUG [hfile-prefetch-1731109750808 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/family/aa1464c143d04cb1ac99f03029035a60, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-08T23:49:15,613 DEBUG [hfile-prefetch-1731109750808 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/family/aa1464c143d04cb1ac99f03029035a60. Will sleep 100ms and try the verification again. 2024-11-08T23:49:15,654 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T23:49:15,654 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T23:49:15,654 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T23:49:15,654 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T23:49:15,658 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T23:49:15,658 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731109755654 2024-11-08T23:49:15,658 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T23:49:15,685 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;a15bf3ddec7f:45685,5,FailOnTimeoutGroup] 2024-11-08T23:49:15,686 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a15bf3ddec7f,45685,1731109752978' ***** 2024-11-08T23:49:15,686 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-08T23:49:15,686 INFO [RS:1;a15bf3ddec7f:45685 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T23:49:15,686 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T23:49:15,686 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T23:49:15,686 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T23:49:15,686 INFO [RS:1;a15bf3ddec7f:45685 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.persistence 2024-11-08T23:49:15,686 INFO [RS:1;a15bf3ddec7f:45685 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T23:49:15,686 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T23:49:15,696 DEBUG [RS:1;a15bf3ddec7f:45685 {}] bucket.BucketCache(1488): Thread RS:1;a15bf3ddec7f:45685 started persisting bucket cache to file 2024-11-08T23:49:15,696 DEBUG [RS:1;a15bf3ddec7f:45685 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T23:49:15,697 DEBUG [RS:1;a15bf3ddec7f:45685 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T23:49:15,700 DEBUG [RS:1;a15bf3ddec7f:45685 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-08T23:49:15,700 DEBUG [RS:1;a15bf3ddec7f:45685 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731109755696 2024-11-08T23:49:15,700 DEBUG [RS:1;a15bf3ddec7f:45685 {}] bucket.BucketCache(1510): Thread RS:1;a15bf3ddec7f:45685 finished persisting bucket cache to file, renaming 2024-11-08T23:49:15,700 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T23:49:15,700 INFO [RS:1;a15bf3ddec7f:45685 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T23:49:15,700 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T23:49:15,700 INFO [RS:1;a15bf3ddec7f:45685 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T23:49:15,700 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(3091): Received CLOSE for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:15,700 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(959): stopping server a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:15,700 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T23:49:15,700 INFO [RS:1;a15bf3ddec7f:45685 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a15bf3ddec7f:45685. 2024-11-08T23:49:15,701 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T23:49:15,701 DEBUG [RS:1;a15bf3ddec7f:45685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T23:49:15,701 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 450433d2ce3191256fcef97bd2b0549e, disabling compactions & flushes 2024-11-08T23:49:15,701 DEBUG [RS:1;a15bf3ddec7f:45685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:15,701 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:15,701 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:15,701 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. after waiting 0 ms 2024-11-08T23:49:15,701 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:15,701 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T23:49:15,701 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T23:49:15,701 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T23:49:15,701 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T23:49:15,701 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-08T23:49:15,701 DEBUG [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(1325): Online Regions={450433d2ce3191256fcef97bd2b0549e=testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e., 1588230740=hbase:meta,,1.1588230740} 2024-11-08T23:49:15,701 DEBUG [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:15,701 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T23:49:15,701 DEBUG [StoreFileCloser-450433d2ce3191256fcef97bd2b0549e-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/family/aa1464c143d04cb1ac99f03029035a60 2024-11-08T23:49:15,701 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T23:49:15,701 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T23:49:15,702 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T23:49:15,702 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T23:49:15,702 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T23:49:15,701 WARN [hfile-prefetch-1731109750808 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/family/aa1464c143d04cb1ac99f03029035a60, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-08T23:49:15,702 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-08T23:49:15,702 DEBUG [hfile-prefetch-1731109750808 {}] hfile.PrefetchExecutor(121): Prefetch completed for aa1464c143d04cb1ac99f03029035a60 2024-11-08T23:49:15,707 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T23:49:15,707 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:15,707 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 450433d2ce3191256fcef97bd2b0549e: Waiting for close lock at 1731109755700Running coprocessor pre-close hooks at 1731109755701 (+1 ms)Disabling compacts and flushes for region at 1731109755701Disabling writes for close at 1731109755701Writing region close event to WAL at 1731109755702 (+1 ms)Running coprocessor post-close hooks at 1731109755707 (+5 ms)Closed at 1731109755707 2024-11-08T23:49:15,707 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:15,712 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ed5ae0ac41a24912ae85506af01a8564 2024-11-08T23:49:15,725 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/info/ed5ae0ac41a24912ae85506af01a8564 is 209, key is testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e./info:regioninfo/1731109754370/Put/seqid=0 2024-11-08T23:49:15,726 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/info/ed5ae0ac41a24912ae85506af01a8564 2024-11-08T23:49:15,734 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5bded725668e471988e26960211810ce 2024-11-08T23:49:15,747 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/ns/5bded725668e471988e26960211810ce is 43, key is default/ns:d/1731109753890/Put/seqid=0 2024-11-08T23:49:15,747 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/ns/5bded725668e471988e26960211810ce 2024-11-08T23:49:15,747 INFO [regionserver/a15bf3ddec7f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T23:49:15,755 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 518c015d9dab40c3adf5b325c2bf4a76 2024-11-08T23:49:15,768 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/table/518c015d9dab40c3adf5b325c2bf4a76 is 80, key is testBlockEvictionOnGracefulStop1731109754007/table:state/1731109754379/Put/seqid=0 2024-11-08T23:49:15,768 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/table/518c015d9dab40c3adf5b325c2bf4a76 2024-11-08T23:49:15,771 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/info/ed5ae0ac41a24912ae85506af01a8564 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/ed5ae0ac41a24912ae85506af01a8564 2024-11-08T23:49:15,772 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/ed5ae0ac41a24912ae85506af01a8564, delay=929 ms 2024-11-08T23:49:15,773 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/ed5ae0ac41a24912ae85506af01a8564, entries=10, sequenceid=11, filesize=7.0 K 2024-11-08T23:49:15,773 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/ns/5bded725668e471988e26960211810ce as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/ns/5bded725668e471988e26960211810ce 2024-11-08T23:49:15,774 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/ns/5bded725668e471988e26960211810ce, delay=992 ms 2024-11-08T23:49:15,775 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/ns/5bded725668e471988e26960211810ce, entries=2, sequenceid=11, filesize=5.0 K 2024-11-08T23:49:15,775 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/table/518c015d9dab40c3adf5b325c2bf4a76 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/table/518c015d9dab40c3adf5b325c2bf4a76 2024-11-08T23:49:15,776 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/table/518c015d9dab40c3adf5b325c2bf4a76, delay=962 ms 2024-11-08T23:49:15,777 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/table/518c015d9dab40c3adf5b325c2bf4a76, entries=2, sequenceid=11, filesize=5.3 K 2024-11-08T23:49:15,778 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=11, compaction requested=false 2024-11-08T23:49:15,778 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/ed5ae0ac41a24912ae85506af01a8564 2024-11-08T23:49:15,779 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/ns/5bded725668e471988e26960211810ce 2024-11-08T23:49:15,780 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/table/518c015d9dab40c3adf5b325c2bf4a76 2024-11-08T23:49:15,784 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-08T23:49:15,784 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T23:49:15,784 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T23:49:15,785 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731109755701Running coprocessor pre-close hooks at 1731109755701Disabling compacts and flushes for region at 1731109755701Disabling writes for close at 1731109755702 (+1 ms)Obtaining lock to block concurrent updates at 1731109755702Preparing flush snapshotting stores in 1588230740 at 1731109755702Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731109755702Flushing stores of hbase:meta,,1.1588230740 at 1731109755703 (+1 ms)Flushing 1588230740/info: creating writer at 1731109755703Flushing 1588230740/info: appending metadata at 1731109755725 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731109755725Flushing 1588230740/ns: creating writer at 1731109755728 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731109755746 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731109755746Flushing 1588230740/table: creating writer at 1731109755749 (+3 ms)Flushing 1588230740/table: appending metadata at 1731109755767 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731109755767Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30f6c381: reopening flushed file at 1731109755771 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66e3ca05: reopening flushed file at 1731109755773 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bdf8537: reopening flushed file at 1731109755775 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=11, compaction requested=false at 1731109755778 (+3 ms)Writing region close event to WAL at 1731109755780 (+2 ms)Running coprocessor post-close hooks at 1731109755784 (+4 ms)Closed at 1731109755784 2024-11-08T23:49:15,785 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T23:49:15,902 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(976): stopping server a15bf3ddec7f,45685,1731109752978; all regions closed. 2024-11-08T23:49:15,903 DEBUG [RS:1;a15bf3ddec7f:45685 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs 2024-11-08T23:49:15,903 INFO [RS:1;a15bf3ddec7f:45685 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a15bf3ddec7f%2C45685%2C1731109752978.meta:.meta(num 1731109753841) 2024-11-08T23:49:15,905 DEBUG [RS:1;a15bf3ddec7f:45685 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs 2024-11-08T23:49:15,905 INFO [RS:1;a15bf3ddec7f:45685 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a15bf3ddec7f%2C45685%2C1731109752978:(num 1731109753487) 2024-11-08T23:49:15,905 DEBUG [RS:1;a15bf3ddec7f:45685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:15,905 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T23:49:15,905 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T23:49:15,905 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.ChoreService(370): Chore service for: regionserver/a15bf3ddec7f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T23:49:15,906 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T23:49:15,906 INFO [regionserver/a15bf3ddec7f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T23:49:15,906 INFO [RS:1;a15bf3ddec7f:45685 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45685 2024-11-08T23:49:15,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a15bf3ddec7f,45685,1731109752978 2024-11-08T23:49:15,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T23:49:15,910 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T23:49:15,913 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a15bf3ddec7f,45685,1731109752978] 2024-11-08T23:49:15,914 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a15bf3ddec7f,45685,1731109752978 already deleted, retry=false 2024-11-08T23:49:15,914 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a15bf3ddec7f,45685,1731109752978 on a15bf3ddec7f,46357,1731109752893 2024-11-08T23:49:15,919 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a15bf3ddec7f,45685,1731109752978, splitWal=true, meta=true 2024-11-08T23:49:15,921 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for a15bf3ddec7f,45685,1731109752978 (carryingMeta=true) a15bf3ddec7f,45685,1731109752978/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@74fa682[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-08T23:49:15,921 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a15bf3ddec7f,45685,1731109752978, splitWal=true, meta=true 2024-11-08T23:49:15,922 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure a15bf3ddec7f,45685,1731109752978, splitWal=true, meta=true, isMeta: true 2024-11-08T23:49:15,923 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,45685,1731109752978-splitting 2024-11-08T23:49:15,923 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,45685,1731109752978-splitting dir is empty, no logs to split. 2024-11-08T23:49:15,923 INFO [PEWorker-2 {}] master.SplitWALManager(105): a15bf3ddec7f,45685,1731109752978 WAL count=0, meta=true 2024-11-08T23:49:15,926 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,45685,1731109752978-splitting dir is empty, no logs to split. 2024-11-08T23:49:15,926 INFO [PEWorker-2 {}] master.SplitWALManager(105): a15bf3ddec7f,45685,1731109752978 WAL count=0, meta=true 2024-11-08T23:49:15,926 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if a15bf3ddec7f,45685,1731109752978 WAL splitting is done? wals=0, meta=true 2024-11-08T23:49:15,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T23:49:15,929 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T23:49:15,930 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T23:49:16,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:16,013 INFO [RS:1;a15bf3ddec7f:45685 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T23:49:16,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45685-0x100f9fc7abc0002, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:16,013 INFO [RS:1;a15bf3ddec7f:45685 {}] regionserver.HRegionServer(1031): Exiting; stopping=a15bf3ddec7f,45685,1731109752978; zookeeper connection closed. 2024-11-08T23:49:16,013 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5219eb2b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5219eb2b 2024-11-08T23:49:16,080 DEBUG [a15bf3ddec7f:46357 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T23:49:16,081 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a15bf3ddec7f,46179,1731109752940 2024-11-08T23:49:16,082 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,46179,1731109752940, state=OPENING 2024-11-08T23:49:16,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T23:49:16,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T23:49:16,083 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T23:49:16,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:16,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:16,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a15bf3ddec7f,46179,1731109752940}] 2024-11-08T23:49:16,202 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a15bf3ddec7f:0 server-side Connection retries=45 2024-11-08T23:49:16,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:16,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:16,202 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T23:49:16,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T23:49:16,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T23:49:16,202 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T23:49:16,202 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T23:49:16,203 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43517 2024-11-08T23:49:16,204 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43517 connecting to ZooKeeper ensemble=127.0.0.1:49539 2024-11-08T23:49:16,205 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:16,205 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:16,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:435170x0, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T23:49:16,208 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43517-0x100f9fc7abc0003 connected 2024-11-08T23:49:16,208 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43517-0x100f9fc7abc0003, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-08T23:49:16,209 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-08T23:49:16,209 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T23:49:16,210 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.cache 2024-11-08T23:49:16,210 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T23:49:16,210 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T23:49:16,210 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T23:49:16,210 INFO [Thread-207 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T23:49:16,211 INFO [Thread-207 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-08T23:49:16,212 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T23:49:16,212 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T23:49:16,212 INFO [Thread-207 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-08T23:49:16,213 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43517-0x100f9fc7abc0003, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T23:49:16,214 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43517-0x100f9fc7abc0003, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T23:49:16,215 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43517 2024-11-08T23:49:16,215 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43517 2024-11-08T23:49:16,215 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43517 2024-11-08T23:49:16,216 DEBUG [Thread-207 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-08T23:49:16,216 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T23:49:16,216 INFO [Thread-207 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-08T23:49:16,216 INFO [Thread-207 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.persistence is complete. 2024-11-08T23:49:16,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43517 2024-11-08T23:49:16,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43517 2024-11-08T23:49:16,221 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(746): ClusterId : ced4ba26-b3e7-47b2-ab2d-bf8c5ffc4f58 2024-11-08T23:49:16,221 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T23:49:16,225 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T23:49:16,225 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T23:49:16,227 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T23:49:16,227 DEBUG [RS:2;a15bf3ddec7f:43517 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73fc9186, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a15bf3ddec7f/172.17.0.2:0 2024-11-08T23:49:16,236 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T23:49:16,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38859, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T23:49:16,240 DEBUG [RS:2;a15bf3ddec7f:43517 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;a15bf3ddec7f:43517 2024-11-08T23:49:16,240 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T23:49:16,240 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T23:49:16,240 DEBUG [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T23:49:16,240 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T23:49:16,240 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T23:49:16,240 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T23:49:16,241 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(2659): reportForDuty to master=a15bf3ddec7f,46357,1731109752893 with port=43517, startcode=1731109756201 2024-11-08T23:49:16,241 DEBUG [RS:2;a15bf3ddec7f:43517 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T23:49:16,241 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a15bf3ddec7f%2C46179%2C1731109752940.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,46179,1731109752940, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs, maxLogs=55 2024-11-08T23:49:16,244 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59789, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T23:49:16,244 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a15bf3ddec7f,43517,1731109756201 2024-11-08T23:49:16,244 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46357 {}] master.ServerManager(517): Registering regionserver=a15bf3ddec7f,43517,1731109756201 2024-11-08T23:49:16,246 DEBUG [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59 2024-11-08T23:49:16,246 DEBUG [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T23:49:16,246 DEBUG [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T23:49:16,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T23:49:16,248 DEBUG [RS:2;a15bf3ddec7f:43517 {}] zookeeper.ZKUtil(111): regionserver:43517-0x100f9fc7abc0003, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a15bf3ddec7f,43517,1731109756201 2024-11-08T23:49:16,248 WARN [RS:2;a15bf3ddec7f:43517 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T23:49:16,249 INFO [RS:2;a15bf3ddec7f:43517 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T23:49:16,249 DEBUG [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,43517,1731109756201 2024-11-08T23:49:16,249 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a15bf3ddec7f,43517,1731109756201] 2024-11-08T23:49:16,254 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T23:49:16,257 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T23:49:16,258 INFO [RS:2;a15bf3ddec7f:43517 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T23:49:16,258 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:16,258 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T23:49:16,258 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T23:49:16,258 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a15bf3ddec7f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a15bf3ddec7f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T23:49:16,259 DEBUG [RS:2;a15bf3ddec7f:43517 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a15bf3ddec7f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T23:49:16,261 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:16,261 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:16,261 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:16,261 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:16,261 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:16,261 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,43517,1731109756201-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T23:49:16,261 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,46179,1731109752940/a15bf3ddec7f%2C46179%2C1731109752940.meta.1731109756241.meta 2024-11-08T23:49:16,261 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T23:49:16,262 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T23:49:16,262 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T23:49:16,262 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T23:49:16,262 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T23:49:16,262 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T23:49:16,262 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:16,263 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T23:49:16,263 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T23:49:16,263 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T23:49:16,264 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T23:49:16,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:16,269 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/ed5ae0ac41a24912ae85506af01a8564, delay=903 ms 2024-11-08T23:49:16,270 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/ed5ae0ac41a24912ae85506af01a8564 2024-11-08T23:49:16,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:16,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T23:49:16,271 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T23:49:16,271 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:16,273 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/ns/5bded725668e471988e26960211810ce, delay=911 ms 2024-11-08T23:49:16,274 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/ns/5bded725668e471988e26960211810ce 2024-11-08T23:49:16,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:16,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T23:49:16,275 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T23:49:16,275 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:16,275 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:16,275 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T23:49:16,276 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T23:49:16,276 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:16,276 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T23:49:16,277 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,43517,1731109756201-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:16,277 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:16,277 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.Replication(171): a15bf3ddec7f,43517,1731109756201 started 2024-11-08T23:49:16,278 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/table/518c015d9dab40c3adf5b325c2bf4a76, delay=980 ms 2024-11-08T23:49:16,279 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/table/518c015d9dab40c3adf5b325c2bf4a76 2024-11-08T23:49:16,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T23:49:16,280 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T23:49:16,280 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740 2024-11-08T23:49:16,281 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740 2024-11-08T23:49:16,281 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T23:49:16,281 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T23:49:16,282 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T23:49:16,283 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T23:49:16,284 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72674482, jitterRate=0.08293417096138}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T23:49:16,284 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T23:49:16,285 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731109756263Writing region info on filesystem at 1731109756263Initializing all the Stores at 1731109756263Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109756263Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109756263Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109756263Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109756263Cleaning up temporary data from old regions at 1731109756281 (+18 ms)Running coprocessor post-open hooks at 1731109756284 (+3 ms)Region opened successfully at 1731109756285 (+1 ms) 2024-11-08T23:49:16,286 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731109756236 2024-11-08T23:49:16,288 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T23:49:16,289 INFO [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T23:49:16,289 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=a15bf3ddec7f,46179,1731109752940 2024-11-08T23:49:16,290 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,46179,1731109752940, state=OPEN 2024-11-08T23:49:16,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T23:49:16,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43517-0x100f9fc7abc0003, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T23:49:16,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T23:49:16,292 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:16,292 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:16,292 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T23:49:16,292 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a15bf3ddec7f,46179,1731109752940 2024-11-08T23:49:16,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-08T23:49:16,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a15bf3ddec7f,46179,1731109752940 in 208 msec 2024-11-08T23:49:16,294 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T23:49:16,294 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(1482): Serving as a15bf3ddec7f,43517,1731109756201, RpcServer on a15bf3ddec7f/172.17.0.2:43517, sessionid=0x100f9fc7abc0003 2024-11-08T23:49:16,294 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T23:49:16,294 DEBUG [RS:2;a15bf3ddec7f:43517 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a15bf3ddec7f,43517,1731109756201 2024-11-08T23:49:16,295 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a15bf3ddec7f,43517,1731109756201' 2024-11-08T23:49:16,295 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T23:49:16,295 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T23:49:16,296 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T23:49:16,296 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T23:49:16,296 DEBUG [RS:2;a15bf3ddec7f:43517 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a15bf3ddec7f,43517,1731109756201 2024-11-08T23:49:16,296 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a15bf3ddec7f,43517,1731109756201' 2024-11-08T23:49:16,296 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T23:49:16,296 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T23:49:16,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T23:49:16,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 366 msec 2024-11-08T23:49:16,297 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): a15bf3ddec7f,45685,1731109752978 had 2 regions 2024-11-08T23:49:16,297 DEBUG [RS:2;a15bf3ddec7f:43517 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T23:49:16,297 INFO [RS:2;a15bf3ddec7f:43517 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T23:49:16,297 INFO [RS:2;a15bf3ddec7f:43517 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T23:49:16,297 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure a15bf3ddec7f,45685,1731109752978, splitWal=true, meta=true, isMeta: false 2024-11-08T23:49:16,298 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,45685,1731109752978-splitting dir is empty, no logs to split. 2024-11-08T23:49:16,298 INFO [PEWorker-3 {}] master.SplitWALManager(105): a15bf3ddec7f,45685,1731109752978 WAL count=0, meta=false 2024-11-08T23:49:16,299 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,45685,1731109752978-splitting dir is empty, no logs to split. 2024-11-08T23:49:16,299 INFO [PEWorker-3 {}] master.SplitWALManager(105): a15bf3ddec7f,45685,1731109752978 WAL count=0, meta=false 2024-11-08T23:49:16,299 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if a15bf3ddec7f,45685,1731109752978 WAL splitting is done? wals=0, meta=false 2024-11-08T23:49:16,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731109754007, region=450433d2ce3191256fcef97bd2b0549e, ASSIGN}] 2024-11-08T23:49:16,301 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731109754007, region=450433d2ce3191256fcef97bd2b0549e, ASSIGN 2024-11-08T23:49:16,301 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731109754007, region=450433d2ce3191256fcef97bd2b0549e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T23:49:16,397 INFO [RS:2;a15bf3ddec7f:43517 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T23:49:16,398 INFO [RS:2;a15bf3ddec7f:43517 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a15bf3ddec7f%2C43517%2C1731109756201, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,43517,1731109756201, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs, maxLogs=55 2024-11-08T23:49:16,417 INFO [RS:2;a15bf3ddec7f:43517 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/WALs/a15bf3ddec7f,43517,1731109756201/a15bf3ddec7f%2C43517%2C1731109756201.1731109756399 2024-11-08T23:49:16,417 DEBUG [RS:2;a15bf3ddec7f:43517 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T23:49:16,452 DEBUG [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(204): Hosts are {a15bf3ddec7f=0} racks are {/default-rack=0} 2024-11-08T23:49:16,452 DEBUG [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T23:49:16,452 DEBUG [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T23:49:16,452 DEBUG [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T23:49:16,452 DEBUG [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T23:49:16,452 INFO [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T23:49:16,452 INFO [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T23:49:16,452 DEBUG [a15bf3ddec7f:46357 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T23:49:16,453 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=450433d2ce3191256fcef97bd2b0549e, regionState=OPENING, regionLocation=a15bf3ddec7f,43517,1731109756201 2024-11-08T23:49:16,454 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a15bf3ddec7f:45685 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a15bf3ddec7f/172.17.0.2:45685 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T23:49:16,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a15bf3ddec7f,45685,1731109752978, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a15bf3ddec7f,45685,1731109752978, seqNum=-1, error=java.net.ConnectException: Call to address=a15bf3ddec7f:45685 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a15bf3ddec7f/172.17.0.2:45685 2024-11-08T23:49:16,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a15bf3ddec7f,45685,1731109752978, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T23:49:16,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a15bf3ddec7f,45685,1731109752978, seqNum=-1 from cache 2024-11-08T23:49:16,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address a15bf3ddec7f:45685 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a15bf3ddec7f/172.17.0.2:45685 2024-11-08T23:49:16,560 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T23:49:16,560 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a15bf3ddec7f,46179,1731109752940, seqNum=-1] 2024-11-08T23:49:16,561 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T23:49:16,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59389, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T23:49:16,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731109754007, region=450433d2ce3191256fcef97bd2b0549e, ASSIGN because future has completed 2024-11-08T23:49:16,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 450433d2ce3191256fcef97bd2b0549e, server=a15bf3ddec7f,43517,1731109756201}] 2024-11-08T23:49:16,718 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T23:49:16,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35849, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T23:49:16,722 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:16,722 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 450433d2ce3191256fcef97bd2b0549e, NAME => 'testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e.', STARTKEY => '', ENDKEY => ''} 2024-11-08T23:49:16,723 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731109754007 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,723 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T23:49:16,723 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,723 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,724 INFO [StoreOpener-450433d2ce3191256fcef97bd2b0549e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,725 INFO [StoreOpener-450433d2ce3191256fcef97bd2b0549e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 450433d2ce3191256fcef97bd2b0549e columnFamilyName family 2024-11-08T23:49:16,725 DEBUG [StoreOpener-450433d2ce3191256fcef97bd2b0549e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T23:49:16,726 DEBUG [StoreFileOpener-450433d2ce3191256fcef97bd2b0549e-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/family/aa1464c143d04cb1ac99f03029035a60, delay=947 ms 2024-11-08T23:49:16,727 DEBUG [StoreOpener-450433d2ce3191256fcef97bd2b0549e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/family/aa1464c143d04cb1ac99f03029035a60 2024-11-08T23:49:16,727 INFO [StoreOpener-450433d2ce3191256fcef97bd2b0549e-1 {}] regionserver.HStore(327): Store=450433d2ce3191256fcef97bd2b0549e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T23:49:16,727 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,728 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,728 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,728 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,729 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,729 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,730 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 450433d2ce3191256fcef97bd2b0549e; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67663408, jitterRate=0.008263349533081055}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T23:49:16,730 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,731 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 450433d2ce3191256fcef97bd2b0549e: Running coprocessor pre-open hook at 1731109756723Writing region info on filesystem at 1731109756723Initializing all the Stores at 1731109756723Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109756723Cleaning up temporary data from old regions at 1731109756729 (+6 ms)Running coprocessor post-open hooks at 1731109756730 (+1 ms)Region opened successfully at 1731109756731 (+1 ms) 2024-11-08T23:49:16,732 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e., pid=11, masterSystemTime=1731109756717 2024-11-08T23:49:16,734 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:16,734 INFO [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:16,734 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=450433d2ce3191256fcef97bd2b0549e, regionState=OPEN, openSeqNum=10, regionLocation=a15bf3ddec7f,43517,1731109756201 2024-11-08T23:49:16,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 450433d2ce3191256fcef97bd2b0549e, server=a15bf3ddec7f,43517,1731109756201 because future has completed 2024-11-08T23:49:16,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-08T23:49:16,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 450433d2ce3191256fcef97bd2b0549e, server=a15bf3ddec7f,43517,1731109756201 in 173 msec 2024-11-08T23:49:16,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-08T23:49:16,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731109754007, region=450433d2ce3191256fcef97bd2b0549e, ASSIGN in 439 msec 2024-11-08T23:49:16,741 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server a15bf3ddec7f,45685,1731109752978 after splitting done 2024-11-08T23:49:16,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure a15bf3ddec7f,45685,1731109752978, splitWal=true, meta=true in 826 msec 2024-11-08T23:49:16,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T23:49:16,795 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T23:49:16,795 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T23:49:16,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:16,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:16,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T23:49:16,795 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T23:49:16,795 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1106274295, stopped=false 2024-11-08T23:49:16,795 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a15bf3ddec7f,46357,1731109752893 2024-11-08T23:49:16,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T23:49:16,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43517-0x100f9fc7abc0003, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T23:49:16,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T23:49:16,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:16,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43517-0x100f9fc7abc0003, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:16,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:16,798 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T23:49:16,798 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T23:49:16,799 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T23:49:16,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:16,799 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T23:49:16,799 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a15bf3ddec7f,46179,1731109752940' ***** 2024-11-08T23:49:16,799 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T23:49:16,799 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a15bf3ddec7f,43517,1731109756201' ***** 2024-11-08T23:49:16,799 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T23:49:16,799 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T23:49:16,799 INFO [RS:0;a15bf3ddec7f:46179 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T23:49:16,799 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43517-0x100f9fc7abc0003, quorum=127.0.0.1:49539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T23:49:16,799 INFO [RS:2;a15bf3ddec7f:43517 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T23:49:16,800 INFO [RS:2;a15bf3ddec7f:43517 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.persistence 2024-11-08T23:49:16,800 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T23:49:16,800 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T23:49:16,800 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T23:49:16,800 INFO [RS:0;a15bf3ddec7f:46179 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27608b39-682c-5882-6349-9cffd4fd803b/bucket.persistence 2024-11-08T23:49:16,800 INFO [RS:2;a15bf3ddec7f:43517 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T23:49:16,800 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T23:49:16,800 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T23:49:16,800 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T23:49:16,800 INFO [RS:0;a15bf3ddec7f:46179 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T23:49:16,800 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T23:49:16,800 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T23:49:16,810 DEBUG [RS:2;a15bf3ddec7f:43517 {}] bucket.BucketCache(1488): Thread RS:2;a15bf3ddec7f:43517 started persisting bucket cache to file 2024-11-08T23:49:16,810 DEBUG [RS:2;a15bf3ddec7f:43517 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T23:49:16,810 DEBUG [RS:2;a15bf3ddec7f:43517 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T23:49:16,810 DEBUG [RS:0;a15bf3ddec7f:46179 {}] bucket.BucketCache(1488): Thread RS:0;a15bf3ddec7f:46179 started persisting bucket cache to file 2024-11-08T23:49:16,811 DEBUG [RS:0;a15bf3ddec7f:46179 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T23:49:16,811 DEBUG [RS:0;a15bf3ddec7f:46179 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T23:49:16,814 DEBUG [RS:0;a15bf3ddec7f:46179 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T23:49:16,814 DEBUG [RS:0;a15bf3ddec7f:46179 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731109756810 2024-11-08T23:49:16,814 DEBUG [RS:2;a15bf3ddec7f:43517 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-08T23:49:16,814 DEBUG [RS:0;a15bf3ddec7f:46179 {}] bucket.BucketCache(1510): Thread RS:0;a15bf3ddec7f:46179 finished persisting bucket cache to file, renaming 2024-11-08T23:49:16,814 DEBUG [RS:2;a15bf3ddec7f:43517 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731109756810 2024-11-08T23:49:16,814 DEBUG [RS:2;a15bf3ddec7f:43517 {}] bucket.BucketCache(1510): Thread RS:2;a15bf3ddec7f:43517 finished persisting bucket cache to file, renaming 2024-11-08T23:49:16,814 WARN [RS:2;a15bf3ddec7f:43517 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-08T23:49:16,814 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T23:49:16,814 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T23:49:16,814 INFO [RS:0;a15bf3ddec7f:46179 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T23:49:16,814 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T23:49:16,814 INFO [RS:2;a15bf3ddec7f:43517 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T23:49:16,814 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T23:49:16,814 INFO [RS:0;a15bf3ddec7f:46179 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T23:49:16,814 INFO [RS:2;a15bf3ddec7f:43517 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T23:49:16,815 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(959): stopping server a15bf3ddec7f,46179,1731109752940 2024-11-08T23:49:16,815 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(3091): Received CLOSE for 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,815 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T23:49:16,815 INFO [RS:0;a15bf3ddec7f:46179 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a15bf3ddec7f:46179. 2024-11-08T23:49:16,815 DEBUG [RS:0;a15bf3ddec7f:46179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T23:49:16,815 DEBUG [RS:0;a15bf3ddec7f:46179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:16,815 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(959): stopping server a15bf3ddec7f,43517,1731109756201 2024-11-08T23:49:16,815 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T23:49:16,815 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T23:49:16,815 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T23:49:16,815 INFO [RS:2;a15bf3ddec7f:43517 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;a15bf3ddec7f:43517. 2024-11-08T23:49:16,815 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T23:49:16,815 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T23:49:16,815 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T23:49:16,815 DEBUG [RS:2;a15bf3ddec7f:43517 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T23:49:16,815 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 450433d2ce3191256fcef97bd2b0549e, disabling compactions & flushes 2024-11-08T23:49:16,815 DEBUG [RS:2;a15bf3ddec7f:43517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:16,815 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:16,815 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:16,815 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T23:49:16,815 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. after waiting 0 ms 2024-11-08T23:49:16,815 DEBUG [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(1325): Online Regions={450433d2ce3191256fcef97bd2b0549e=testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e.} 2024-11-08T23:49:16,815 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:16,815 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T23:49:16,816 DEBUG [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(1351): Waiting on 450433d2ce3191256fcef97bd2b0549e 2024-11-08T23:49:16,816 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T23:49:16,816 DEBUG [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T23:49:16,816 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T23:49:16,816 DEBUG [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T23:49:16,816 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T23:49:16,816 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T23:49:16,816 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T23:49:16,816 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T23:49:16,816 DEBUG [StoreFileCloser-450433d2ce3191256fcef97bd2b0549e-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/family/aa1464c143d04cb1ac99f03029035a60 2024-11-08T23:49:16,816 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-08T23:49:16,823 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/default/testBlockEvictionOnGracefulStop1731109754007/450433d2ce3191256fcef97bd2b0549e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-08T23:49:16,824 INFO [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:16,824 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 450433d2ce3191256fcef97bd2b0549e: Waiting for close lock at 1731109756815Running coprocessor pre-close hooks at 1731109756815Disabling compacts and flushes for region at 1731109756815Disabling writes for close at 1731109756815Writing region close event to WAL at 1731109756817 (+2 ms)Running coprocessor post-close hooks at 1731109756823 (+6 ms)Closed at 1731109756823 2024-11-08T23:49:16,824 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e. 2024-11-08T23:49:16,828 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cd5161d36ee2425a8e8a1739288c9368 2024-11-08T23:49:16,841 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/info/cd5161d36ee2425a8e8a1739288c9368 is 209, key is testBlockEvictionOnGracefulStop1731109754007,,1731109754011.450433d2ce3191256fcef97bd2b0549e./info:regioninfo/1731109756734/Put/seqid=0 2024-11-08T23:49:16,841 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/info/cd5161d36ee2425a8e8a1739288c9368 2024-11-08T23:49:16,842 INFO [regionserver/a15bf3ddec7f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T23:49:16,844 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/.tmp/info/cd5161d36ee2425a8e8a1739288c9368 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/cd5161d36ee2425a8e8a1739288c9368 2024-11-08T23:49:16,845 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/cd5161d36ee2425a8e8a1739288c9368, delay=984 ms 2024-11-08T23:49:16,846 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/cd5161d36ee2425a8e8a1739288c9368, entries=8, sequenceid=19, filesize=6.6 K 2024-11-08T23:49:16,846 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=19, compaction requested=false 2024-11-08T23:49:16,847 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/ed5ae0ac41a24912ae85506af01a8564 2024-11-08T23:49:16,847 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/info/cd5161d36ee2425a8e8a1739288c9368 2024-11-08T23:49:16,848 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/ns/5bded725668e471988e26960211810ce 2024-11-08T23:49:16,848 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/table/518c015d9dab40c3adf5b325c2bf4a76 2024-11-08T23:49:16,853 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-08T23:49:16,854 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T23:49:16,854 INFO [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T23:49:16,854 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731109756815Running coprocessor pre-close hooks at 1731109756816 (+1 ms)Disabling compacts and flushes for region at 1731109756816Disabling writes for close at 1731109756816Obtaining lock to block concurrent updates at 1731109756816Preparing flush snapshotting stores in 1588230740 at 1731109756816Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731109756817 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731109756817Flushing 1588230740/info: creating writer at 1731109756817Flushing 1588230740/info: appending metadata at 1731109756840 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731109756840Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@569ad0a8: reopening flushed file at 1731109756843 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=19, compaction requested=false at 1731109756846 (+3 ms)Writing region close event to WAL at 1731109756849 (+3 ms)Running coprocessor post-close hooks at 1731109756854 (+5 ms)Closed at 1731109756854 2024-11-08T23:49:16,854 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T23:49:16,862 INFO [regionserver/a15bf3ddec7f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T23:49:17,016 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(976): stopping server a15bf3ddec7f,43517,1731109756201; all regions closed. 2024-11-08T23:49:17,016 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(976): stopping server a15bf3ddec7f,46179,1731109752940; all regions closed. 2024-11-08T23:49:17,018 DEBUG [RS:0;a15bf3ddec7f:46179 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs 2024-11-08T23:49:17,018 DEBUG [RS:2;a15bf3ddec7f:43517 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs 2024-11-08T23:49:17,018 INFO [RS:0;a15bf3ddec7f:46179 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a15bf3ddec7f%2C46179%2C1731109752940.meta:.meta(num 1731109756241) 2024-11-08T23:49:17,018 INFO [RS:2;a15bf3ddec7f:43517 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a15bf3ddec7f%2C43517%2C1731109756201:(num 1731109756399) 2024-11-08T23:49:17,018 DEBUG [RS:2;a15bf3ddec7f:43517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:17,018 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T23:49:17,018 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T23:49:17,018 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.ChoreService(370): Chore service for: regionserver/a15bf3ddec7f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T23:49:17,019 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T23:49:17,019 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T23:49:17,019 INFO [regionserver/a15bf3ddec7f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T23:49:17,019 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T23:49:17,019 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T23:49:17,019 INFO [RS:2;a15bf3ddec7f:43517 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43517 2024-11-08T23:49:17,020 DEBUG [RS:0;a15bf3ddec7f:46179 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/oldWALs 2024-11-08T23:49:17,020 INFO [RS:0;a15bf3ddec7f:46179 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a15bf3ddec7f%2C46179%2C1731109752940:(num 1731109753477) 2024-11-08T23:49:17,020 DEBUG [RS:0;a15bf3ddec7f:46179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T23:49:17,020 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T23:49:17,020 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T23:49:17,021 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.ChoreService(370): Chore service for: regionserver/a15bf3ddec7f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T23:49:17,021 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T23:49:17,021 INFO [regionserver/a15bf3ddec7f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T23:49:17,021 INFO [RS:0;a15bf3ddec7f:46179 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46179 2024-11-08T23:49:17,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43517-0x100f9fc7abc0003, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a15bf3ddec7f,43517,1731109756201 2024-11-08T23:49:17,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T23:49:17,022 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T23:49:17,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a15bf3ddec7f,46179,1731109752940 2024-11-08T23:49:17,023 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T23:49:17,023 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a15bf3ddec7f,43517,1731109756201] 2024-11-08T23:49:17,026 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a15bf3ddec7f,43517,1731109756201 already deleted, retry=false 2024-11-08T23:49:17,026 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a15bf3ddec7f,43517,1731109756201 expired; onlineServers=1 2024-11-08T23:49:17,026 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a15bf3ddec7f,46179,1731109752940] 2024-11-08T23:49:17,028 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a15bf3ddec7f,46179,1731109752940 already deleted, retry=false 2024-11-08T23:49:17,028 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a15bf3ddec7f,46179,1731109752940 expired; onlineServers=0 2024-11-08T23:49:17,028 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a15bf3ddec7f,46357,1731109752893' ***** 2024-11-08T23:49:17,028 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T23:49:17,028 INFO [M:0;a15bf3ddec7f:46357 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T23:49:17,028 INFO [M:0;a15bf3ddec7f:46357 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T23:49:17,028 DEBUG [M:0;a15bf3ddec7f:46357 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T23:49:17,028 DEBUG [M:0;a15bf3ddec7f:46357 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T23:49:17,028 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T23:49:17,028 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.large.0-1731109753221 {}] cleaner.HFileCleaner(306): Exit Thread[master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.large.0-1731109753221,5,FailOnTimeoutGroup] 2024-11-08T23:49:17,028 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.small.0-1731109753224 {}] cleaner.HFileCleaner(306): Exit Thread[master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.small.0-1731109753224,5,FailOnTimeoutGroup] 2024-11-08T23:49:17,029 INFO [M:0;a15bf3ddec7f:46357 {}] hbase.ChoreService(370): Chore service for: master/a15bf3ddec7f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T23:49:17,029 INFO [M:0;a15bf3ddec7f:46357 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T23:49:17,029 DEBUG [M:0;a15bf3ddec7f:46357 {}] master.HMaster(1795): Stopping service threads 2024-11-08T23:49:17,029 INFO [M:0;a15bf3ddec7f:46357 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T23:49:17,029 INFO [M:0;a15bf3ddec7f:46357 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T23:49:17,029 INFO [M:0;a15bf3ddec7f:46357 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T23:49:17,029 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T23:49:17,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T23:49:17,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T23:49:17,030 DEBUG [M:0;a15bf3ddec7f:46357 {}] zookeeper.ZKUtil(347): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T23:49:17,030 WARN [M:0;a15bf3ddec7f:46357 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T23:49:17,031 INFO [M:0;a15bf3ddec7f:46357 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/.lastflushedseqids 2024-11-08T23:49:17,034 INFO [M:0;a15bf3ddec7f:46357 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T23:49:17,034 INFO [M:0;a15bf3ddec7f:46357 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T23:49:17,035 DEBUG [M:0;a15bf3ddec7f:46357 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T23:49:17,035 INFO [M:0;a15bf3ddec7f:46357 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:17,035 DEBUG [M:0;a15bf3ddec7f:46357 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:17,035 DEBUG [M:0;a15bf3ddec7f:46357 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T23:49:17,035 DEBUG [M:0;a15bf3ddec7f:46357 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:17,035 INFO [M:0;a15bf3ddec7f:46357 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-08T23:49:17,046 DEBUG [M:0;a15bf3ddec7f:46357 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 446e835179e3409a9c086964a3937532 2024-11-08T23:49:17,060 DEBUG [M:0;a15bf3ddec7f:46357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/446e835179e3409a9c086964a3937532 is 82, key is hbase:meta,,1/info:regioninfo/1731109756289/Put/seqid=0 2024-11-08T23:49:17,060 INFO [M:0;a15bf3ddec7f:46357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/446e835179e3409a9c086964a3937532 2024-11-08T23:49:17,069 DEBUG [M:0;a15bf3ddec7f:46357 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c537d50a740c4f00936437d8eb0f0b16 2024-11-08T23:49:17,083 DEBUG [M:0;a15bf3ddec7f:46357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c537d50a740c4f00936437d8eb0f0b16 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731109754382/Put/seqid=0 2024-11-08T23:49:17,083 INFO [M:0;a15bf3ddec7f:46357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c537d50a740c4f00936437d8eb0f0b16 2024-11-08T23:49:17,091 DEBUG [M:0;a15bf3ddec7f:46357 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 01c8dd58b52f4601b58f52cdb4a23f24 2024-11-08T23:49:17,105 DEBUG [M:0;a15bf3ddec7f:46357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/01c8dd58b52f4601b58f52cdb4a23f24 is 69, key is a15bf3ddec7f,43517,1731109756201/rs:state/1731109756245/Put/seqid=0 2024-11-08T23:49:17,105 INFO [M:0;a15bf3ddec7f:46357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/01c8dd58b52f4601b58f52cdb4a23f24 2024-11-08T23:49:17,107 INFO [M:0;a15bf3ddec7f:46357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01c8dd58b52f4601b58f52cdb4a23f24 2024-11-08T23:49:17,107 DEBUG [M:0;a15bf3ddec7f:46357 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/446e835179e3409a9c086964a3937532 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/446e835179e3409a9c086964a3937532 2024-11-08T23:49:17,109 INFO [M:0;a15bf3ddec7f:46357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/446e835179e3409a9c086964a3937532, entries=14, sequenceid=102, filesize=5.9 K 2024-11-08T23:49:17,109 DEBUG [M:0;a15bf3ddec7f:46357 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c537d50a740c4f00936437d8eb0f0b16 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c537d50a740c4f00936437d8eb0f0b16 2024-11-08T23:49:17,111 INFO [M:0;a15bf3ddec7f:46357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c537d50a740c4f00936437d8eb0f0b16, entries=11, sequenceid=102, filesize=6.5 K 2024-11-08T23:49:17,111 DEBUG [M:0;a15bf3ddec7f:46357 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/01c8dd58b52f4601b58f52cdb4a23f24 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/01c8dd58b52f4601b58f52cdb4a23f24 2024-11-08T23:49:17,113 INFO [M:0;a15bf3ddec7f:46357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01c8dd58b52f4601b58f52cdb4a23f24 2024-11-08T23:49:17,113 INFO [M:0;a15bf3ddec7f:46357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52572fc9-e04c-ed89-6fc4-03a880351574/d94e117e-370d-6d62-7ad0-f05bab22ff59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/01c8dd58b52f4601b58f52cdb4a23f24, entries=3, sequenceid=102, filesize=5.3 K 2024-11-08T23:49:17,114 INFO [M:0;a15bf3ddec7f:46357 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=102, compaction requested=false 2024-11-08T23:49:17,115 INFO [M:0;a15bf3ddec7f:46357 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T23:49:17,115 DEBUG [M:0;a15bf3ddec7f:46357 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731109757035Disabling compacts and flushes for region at 1731109757035Disabling writes for close at 1731109757035Obtaining lock to block concurrent updates at 1731109757035Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731109757035Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1731109757035Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731109757036 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731109757036Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731109757059 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731109757059Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731109757062 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731109757082 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731109757082Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731109757085 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731109757104 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731109757104Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@473473: reopening flushed file at 1731109757107 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f594cbe: reopening flushed file at 1731109757109 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@376bedf8: reopening flushed file at 1731109757111 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=102, compaction requested=false at 1731109757114 (+3 ms)Writing region close event to WAL at 1731109757115 (+1 ms)Closed at 1731109757115 2024-11-08T23:49:17,117 INFO [M:0;a15bf3ddec7f:46357 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T23:49:17,117 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T23:49:17,117 INFO [M:0;a15bf3ddec7f:46357 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46357 2024-11-08T23:49:17,117 INFO [M:0;a15bf3ddec7f:46357 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T23:49:17,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43517-0x100f9fc7abc0003, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:17,125 INFO [RS:2;a15bf3ddec7f:43517 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T23:49:17,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43517-0x100f9fc7abc0003, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:17,125 INFO [RS:2;a15bf3ddec7f:43517 {}] regionserver.HRegionServer(1031): Exiting; stopping=a15bf3ddec7f,43517,1731109756201; zookeeper connection closed. 2024-11-08T23:49:17,125 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@321ff7cc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@321ff7cc 2024-11-08T23:49:17,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:17,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46179-0x100f9fc7abc0001, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:17,126 INFO [RS:0;a15bf3ddec7f:46179 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T23:49:17,126 INFO [RS:0;a15bf3ddec7f:46179 {}] regionserver.HRegionServer(1031): Exiting; stopping=a15bf3ddec7f,46179,1731109752940; zookeeper connection closed. 2024-11-08T23:49:17,127 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@431ea781 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@431ea781 2024-11-08T23:49:17,127 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-08T23:49:17,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:17,220 INFO [M:0;a15bf3ddec7f:46357 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T23:49:17,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46357-0x100f9fc7abc0000, quorum=127.0.0.1:49539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T23:49:17,223 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T23:49:17,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T23:49:17,224 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T23:49:17,233 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1606 (was 1606), ProcessCount=11 (was 11), AvailableMemoryMB=3235 (was 3318)