2024-11-24 18:45:32,125 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 18:45:32,145 main DEBUG Took 0.018704 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 18:45:32,146 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 18:45:32,146 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 18:45:32,147 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 18:45:32,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,156 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 18:45:32,168 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,169 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,170 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,170 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,171 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,171 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,172 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,172 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,172 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,173 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,174 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,174 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,175 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,176 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,176 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,177 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,177 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,177 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,178 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,178 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,179 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,180 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 18:45:32,180 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,181 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 18:45:32,182 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 18:45:32,184 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 18:45:32,186 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 18:45:32,187 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 18:45:32,188 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 18:45:32,189 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 18:45:32,200 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 18:45:32,203 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 18:45:32,205 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 18:45:32,205 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 18:45:32,206 main DEBUG createAppenders(={Console}) 2024-11-24 18:45:32,207 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-24 18:45:32,207 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 18:45:32,207 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-24 18:45:32,208 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 18:45:32,208 main DEBUG OutputStream closed 2024-11-24 18:45:32,209 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 18:45:32,209 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 18:45:32,209 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-24 18:45:32,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 18:45:32,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 18:45:32,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 18:45:32,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 18:45:32,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 18:45:32,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 18:45:32,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 18:45:32,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 18:45:32,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 18:45:32,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 18:45:32,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 18:45:32,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 18:45:32,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 18:45:32,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 18:45:32,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 18:45:32,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 18:45:32,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 18:45:32,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 18:45:32,294 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 18:45:32,294 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-24 18:45:32,294 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 18:45:32,295 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-24T18:45:32,306 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-24 18:45:32,309 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 18:45:32,309 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T18:45:32,597 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854 2024-11-24T18:45:32,620 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=511, ProcessCount=11, AvailableMemoryMB=9344 2024-11-24T18:45:32,708 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T18:45:32,814 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:32,828 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/cluster_6dc27073-b6ea-7acc-2bc3-29685eda7a68, deleteOnExit=true 2024-11-24T18:45:32,923 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/cluster_6dc27073-b6ea-7acc-2bc3-29685eda7a68/zookeeper_0, clientPort=58182, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/cluster_6dc27073-b6ea-7acc-2bc3-29685eda7a68/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/cluster_6dc27073-b6ea-7acc-2bc3-29685eda7a68/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T18:45:32,937 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58182 2024-11-24T18:45:32,938 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:33,028 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9 with version=8 2024-11-24T18:45:33,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/hbase-staging 2024-11-24T18:45:33,120 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T18:45:33,358 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f2b92657890a:0 server-side Connection retries=45 2024-11-24T18:45:33,368 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:33,369 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:33,378 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T18:45:33,378 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:33,379 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T18:45:33,556 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T18:45:33,668 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T18:45:33,728 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T18:45:33,741 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T18:45:33,746 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T18:45:33,785 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15560 (auto-detected) 2024-11-24T18:45:33,786 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T18:45:33,807 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45369 2024-11-24T18:45:33,833 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45369 connecting to ZooKeeper ensemble=127.0.0.1:58182 2024-11-24T18:45:33,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:453690x0, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T18:45:33,930 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45369-0x1016e2a480d0000 connected 2024-11-24T18:45:34,038 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:34,039 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:34,047 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T18:45:34,052 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9, hbase.cluster.distributed=false 2024-11-24T18:45:34,074 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T18:45:34,079 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45369 2024-11-24T18:45:34,079 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45369 2024-11-24T18:45:34,080 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45369 2024-11-24T18:45:34,080 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45369 2024-11-24T18:45:34,081 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45369 2024-11-24T18:45:34,159 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f2b92657890a:0 server-side Connection retries=45 2024-11-24T18:45:34,161 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:34,162 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:34,162 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T18:45:34,163 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:34,163 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T18:45:34,166 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T18:45:34,170 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T18:45:34,171 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35473 2024-11-24T18:45:34,174 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35473 connecting to ZooKeeper ensemble=127.0.0.1:58182 2024-11-24T18:45:34,175 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:34,176 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:34,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354730x0, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T18:45:34,191 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:354730x0, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T18:45:34,191 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35473-0x1016e2a480d0001 connected 2024-11-24T18:45:34,196 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T18:45:34,208 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/bucket.cache 2024-11-24T18:45:34,213 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T18:45:34,214 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T18:45:34,216 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T18:45:34,217 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T18:45:34,218 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T18:45:34,224 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T18:45:34,229 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T18:45:34,229 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/bucket.persistence is complete. 2024-11-24T18:45:34,231 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T18:45:34,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T18:45:34,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T18:45:34,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35473 2024-11-24T18:45:34,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35473 2024-11-24T18:45:34,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35473 2024-11-24T18:45:34,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35473 2024-11-24T18:45:34,243 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35473 2024-11-24T18:45:34,261 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f2b92657890a:0 server-side Connection retries=45 2024-11-24T18:45:34,261 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:34,261 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:34,261 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T18:45:34,261 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:34,262 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T18:45:34,262 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T18:45:34,262 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T18:45:34,263 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38015 2024-11-24T18:45:34,265 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38015 connecting to ZooKeeper ensemble=127.0.0.1:58182 2024-11-24T18:45:34,266 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:34,267 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:34,282 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380150x0, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T18:45:34,283 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38015-0x1016e2a480d0002 connected 2024-11-24T18:45:34,283 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T18:45:34,284 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T18:45:34,285 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/bucket.cache 2024-11-24T18:45:34,285 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T18:45:34,286 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T18:45:34,286 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T18:45:34,287 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T18:45:34,287 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T18:45:34,287 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T18:45:34,288 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T18:45:34,288 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T18:45:34,288 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/bucket.persistence is complete. 2024-11-24T18:45:34,290 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T18:45:34,292 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T18:45:34,295 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38015 2024-11-24T18:45:34,295 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38015 2024-11-24T18:45:34,297 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38015 2024-11-24T18:45:34,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38015 2024-11-24T18:45:34,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38015 2024-11-24T18:45:34,323 DEBUG [M:0;f2b92657890a:45369 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f2b92657890a:45369 2024-11-24T18:45:34,324 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f2b92657890a,45369,1732473933194 2024-11-24T18:45:34,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T18:45:34,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T18:45:34,338 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T18:45:34,342 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f2b92657890a,45369,1732473933194 2024-11-24T18:45:34,380 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T18:45:34,380 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:34,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:34,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T18:45:34,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:34,386 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T18:45:34,387 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f2b92657890a,45369,1732473933194 from backup master directory 2024-11-24T18:45:34,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f2b92657890a,45369,1732473933194 2024-11-24T18:45:34,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T18:45:34,401 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T18:45:34,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T18:45:34,402 WARN [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T18:45:34,403 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f2b92657890a,45369,1732473933194 2024-11-24T18:45:34,404 INFO [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T18:45:34,406 INFO [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T18:45:34,414 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/hbase.id] with ID: f3e3649e-c61b-41b2-9411-3090f0cf9945 2024-11-24T18:45:34,414 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/.tmp/hbase.id 2024-11-24T18:45:34,419 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/hbase.id] 2024-11-24T18:45:34,452 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:34,459 INFO [master/f2b92657890a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T18:45:34,479 INFO [master/f2b92657890a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-24T18:45:34,493 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:34,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:34,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:34,534 INFO [master/f2b92657890a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T18:45:34,537 INFO [master/f2b92657890a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T18:45:34,606 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T18:45:34,615 INFO [master/f2b92657890a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T18:45:34,722 INFO [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store 2024-11-24T18:45:34,744 INFO [master/f2b92657890a:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T18:45:34,748 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:34,750 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T18:45:34,751 INFO [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:34,751 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:34,753 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T18:45:34,754 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:34,754 INFO [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:34,755 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732473934750Disabling compacts and flushes for region at 1732473934750Disabling writes for close at 1732473934753 (+3 ms)Writing region close event to WAL at 1732473934754 (+1 ms)Closed at 1732473934754 2024-11-24T18:45:34,755 WARN [master/f2b92657890a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/.initializing 2024-11-24T18:45:34,756 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/WALs/f2b92657890a,45369,1732473933194 2024-11-24T18:45:34,769 INFO [master/f2b92657890a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T18:45:34,785 INFO [master/f2b92657890a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f2b92657890a%2C45369%2C1732473933194, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/WALs/f2b92657890a,45369,1732473933194, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/oldWALs, maxLogs=10 2024-11-24T18:45:34,821 INFO [master/f2b92657890a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/WALs/f2b92657890a,45369,1732473933194/f2b92657890a%2C45369%2C1732473933194.1732473934790 2024-11-24T18:45:34,822 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T18:45:34,822 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T18:45:34,823 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:34,826 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:34,828 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:34,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:34,895 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T18:45:34,900 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:34,902 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:34,903 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:34,908 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T18:45:34,908 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:34,909 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T18:45:34,909 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:34,914 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T18:45:34,914 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:34,915 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T18:45:34,915 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:34,920 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T18:45:34,920 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:34,921 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T18:45:34,921 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:34,925 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:34,925 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:34,928 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:34,929 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:34,932 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T18:45:34,934 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:34,941 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T18:45:34,942 INFO [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70900674, jitterRate=0.05650237202644348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T18:45:34,950 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732473934840Initializing all the Stores at 1732473934840Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473934841 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473934841Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473934842 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473934842Cleaning up temporary data from old regions at 1732473934929 (+87 ms)Region opened successfully at 1732473934950 (+21 ms) 2024-11-24T18:45:34,952 INFO [master/f2b92657890a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T18:45:34,986 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f9dad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f2b92657890a/172.17.0.2:0 2024-11-24T18:45:35,020 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T18:45:35,033 INFO [master/f2b92657890a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T18:45:35,033 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T18:45:35,036 INFO [master/f2b92657890a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T18:45:35,037 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T18:45:35,043 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T18:45:35,044 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T18:45:35,070 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T18:45:35,081 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T18:45:35,127 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T18:45:35,130 INFO [master/f2b92657890a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T18:45:35,132 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T18:45:35,145 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T18:45:35,147 INFO [master/f2b92657890a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T18:45:35,151 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T18:45:35,158 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T18:45:35,160 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T18:45:35,169 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T18:45:35,191 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T18:45:35,200 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T18:45:35,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T18:45:35,222 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T18:45:35,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T18:45:35,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:35,222 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:35,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:35,226 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f2b92657890a,45369,1732473933194, sessionid=0x1016e2a480d0000, setting cluster-up flag (Was=false) 2024-11-24T18:45:35,253 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:35,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:35,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:35,285 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T18:45:35,289 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f2b92657890a,45369,1732473933194 2024-11-24T18:45:35,317 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:35,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:35,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:35,348 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T18:45:35,351 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f2b92657890a,45369,1732473933194 2024-11-24T18:45:35,358 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T18:45:35,406 INFO [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(746): ClusterId : f3e3649e-c61b-41b2-9411-3090f0cf9945 2024-11-24T18:45:35,406 INFO [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(746): ClusterId : f3e3649e-c61b-41b2-9411-3090f0cf9945 2024-11-24T18:45:35,408 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T18:45:35,408 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T18:45:35,427 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T18:45:35,434 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T18:45:35,434 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T18:45:35,434 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T18:45:35,434 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T18:45:35,438 INFO [master/f2b92657890a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T18:45:35,444 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T18:45:35,444 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T18:45:35,445 DEBUG [RS:0;f2b92657890a:35473 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370d240f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f2b92657890a/172.17.0.2:0 2024-11-24T18:45:35,445 DEBUG [RS:1;f2b92657890a:38015 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309cd123, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f2b92657890a/172.17.0.2:0 2024-11-24T18:45:35,446 INFO [master/f2b92657890a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T18:45:35,453 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f2b92657890a,45369,1732473933194 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T18:45:35,463 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f2b92657890a:0, corePoolSize=5, maxPoolSize=5 2024-11-24T18:45:35,464 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f2b92657890a:0, corePoolSize=5, maxPoolSize=5 2024-11-24T18:45:35,464 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f2b92657890a:0, corePoolSize=5, maxPoolSize=5 2024-11-24T18:45:35,464 DEBUG [RS:1;f2b92657890a:38015 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f2b92657890a:38015 2024-11-24T18:45:35,464 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f2b92657890a:0, corePoolSize=5, maxPoolSize=5 2024-11-24T18:45:35,465 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f2b92657890a:0, corePoolSize=10, maxPoolSize=10 2024-11-24T18:45:35,465 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,465 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f2b92657890a:0, corePoolSize=2, maxPoolSize=2 2024-11-24T18:45:35,465 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,468 INFO [RS:1;f2b92657890a:38015 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T18:45:35,468 INFO [RS:1;f2b92657890a:38015 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T18:45:35,468 DEBUG [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T18:45:35,469 DEBUG [RS:0;f2b92657890a:35473 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f2b92657890a:35473 2024-11-24T18:45:35,469 INFO [RS:0;f2b92657890a:35473 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T18:45:35,469 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732473965469 2024-11-24T18:45:35,470 INFO [RS:0;f2b92657890a:35473 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T18:45:35,470 DEBUG [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T18:45:35,478 INFO [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(2659): reportForDuty to master=f2b92657890a,45369,1732473933194 with port=38015, startcode=1732473934260 2024-11-24T18:45:35,478 INFO [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(2659): reportForDuty to master=f2b92657890a,45369,1732473933194 with port=35473, startcode=1732473934123 2024-11-24T18:45:35,478 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T18:45:35,480 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T18:45:35,480 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T18:45:35,480 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T18:45:35,484 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T18:45:35,484 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T18:45:35,485 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T18:45:35,485 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T18:45:35,486 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:35,486 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T18:45:35,487 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,492 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T18:45:35,493 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T18:45:35,494 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T18:45:35,494 DEBUG [RS:0;f2b92657890a:35473 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T18:45:35,494 DEBUG [RS:1;f2b92657890a:38015 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T18:45:35,496 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T18:45:35,497 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T18:45:35,499 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f2b92657890a:0:becomeActiveMaster-HFileCleaner.large.0-1732473935498,5,FailOnTimeoutGroup] 2024-11-24T18:45:35,504 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f2b92657890a:0:becomeActiveMaster-HFileCleaner.small.0-1732473935499,5,FailOnTimeoutGroup] 2024-11-24T18:45:35,504 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,505 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T18:45:35,506 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,507 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,521 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T18:45:35,522 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9 2024-11-24T18:45:35,538 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:35,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T18:45:35,550 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T18:45:35,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:35,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:35,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T18:45:35,556 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T18:45:35,556 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:35,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:35,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T18:45:35,561 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T18:45:35,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:35,562 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:35,562 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T18:45:35,566 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T18:45:35,567 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:35,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:35,567 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T18:45:35,568 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740 2024-11-24T18:45:35,568 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740 2024-11-24T18:45:35,569 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T18:45:35,569 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T18:45:35,569 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T18:45:35,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T18:45:35,571 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43949, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T18:45:35,571 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49849, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T18:45:35,578 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45369 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f2b92657890a,38015,1732473934260 2024-11-24T18:45:35,581 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45369 {}] master.ServerManager(517): Registering regionserver=f2b92657890a,38015,1732473934260 2024-11-24T18:45:35,583 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T18:45:35,583 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68032556, jitterRate=0.01376408338546753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T18:45:35,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732473935539Initializing all the Stores at 1732473935540 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473935541 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473935541Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473935541Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473935541Cleaning up temporary data from old regions at 1732473935569 (+28 ms)Region opened successfully at 1732473935589 (+20 ms) 2024-11-24T18:45:35,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T18:45:35,590 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T18:45:35,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T18:45:35,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T18:45:35,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T18:45:35,593 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T18:45:35,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732473935590Disabling compacts and flushes for region at 1732473935590Disabling writes for close at 1732473935590Writing region close event to WAL at 1732473935593 (+3 ms)Closed at 1732473935593 2024-11-24T18:45:35,594 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45369 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f2b92657890a,35473,1732473934123 2024-11-24T18:45:35,595 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45369 {}] master.ServerManager(517): Registering regionserver=f2b92657890a,35473,1732473934123 2024-11-24T18:45:35,597 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T18:45:35,597 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T18:45:35,599 DEBUG [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9 2024-11-24T18:45:35,599 DEBUG [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T18:45:35,599 DEBUG [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T18:45:35,600 DEBUG [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9 2024-11-24T18:45:35,600 DEBUG [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T18:45:35,600 DEBUG [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T18:45:35,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T18:45:35,617 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T18:45:35,620 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T18:45:35,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T18:45:35,648 DEBUG [RS:0;f2b92657890a:35473 {}] zookeeper.ZKUtil(111): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f2b92657890a,35473,1732473934123 2024-11-24T18:45:35,649 WARN [RS:0;f2b92657890a:35473 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T18:45:35,649 INFO [RS:0;f2b92657890a:35473 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T18:45:35,649 DEBUG [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,35473,1732473934123 2024-11-24T18:45:35,650 DEBUG [RS:1;f2b92657890a:38015 {}] zookeeper.ZKUtil(111): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f2b92657890a,38015,1732473934260 2024-11-24T18:45:35,650 WARN [RS:1;f2b92657890a:38015 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T18:45:35,650 INFO [RS:1;f2b92657890a:38015 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T18:45:35,650 DEBUG [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,38015,1732473934260 2024-11-24T18:45:35,653 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f2b92657890a,38015,1732473934260] 2024-11-24T18:45:35,654 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f2b92657890a,35473,1732473934123] 2024-11-24T18:45:35,689 INFO [RS:0;f2b92657890a:35473 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T18:45:35,689 INFO [RS:1;f2b92657890a:38015 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T18:45:35,722 INFO [RS:1;f2b92657890a:38015 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T18:45:35,722 INFO [RS:0;f2b92657890a:35473 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T18:45:35,728 INFO [RS:0;f2b92657890a:35473 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T18:45:35,728 INFO [RS:1;f2b92657890a:38015 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T18:45:35,729 INFO [RS:0;f2b92657890a:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,729 INFO [RS:1;f2b92657890a:38015 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,733 INFO [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T18:45:35,735 INFO [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T18:45:35,742 INFO [RS:1;f2b92657890a:38015 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T18:45:35,744 INFO [RS:1;f2b92657890a:38015 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,745 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,745 INFO [RS:0;f2b92657890a:35473 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T18:45:35,745 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,745 INFO [RS:0;f2b92657890a:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,746 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,746 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,746 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,746 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,746 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,746 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,746 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f2b92657890a:0, corePoolSize=2, maxPoolSize=2 2024-11-24T18:45:35,746 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,746 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,746 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,747 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,747 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f2b92657890a:0, corePoolSize=2, maxPoolSize=2 2024-11-24T18:45:35,747 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,747 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,747 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,747 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,747 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,747 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,747 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,747 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,748 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f2b92657890a:0, corePoolSize=3, maxPoolSize=3 2024-11-24T18:45:35,748 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,748 DEBUG [RS:1;f2b92657890a:38015 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f2b92657890a:0, corePoolSize=3, maxPoolSize=3 2024-11-24T18:45:35,748 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:35,748 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f2b92657890a:0, corePoolSize=3, maxPoolSize=3 2024-11-24T18:45:35,748 DEBUG [RS:0;f2b92657890a:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f2b92657890a:0, corePoolSize=3, maxPoolSize=3 2024-11-24T18:45:35,752 INFO [RS:1;f2b92657890a:38015 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,753 INFO [RS:1;f2b92657890a:38015 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,753 INFO [RS:1;f2b92657890a:38015 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,753 INFO [RS:1;f2b92657890a:38015 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,754 INFO [RS:1;f2b92657890a:38015 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,754 INFO [RS:1;f2b92657890a:38015 {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,38015,1732473934260-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T18:45:35,755 INFO [RS:0;f2b92657890a:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,755 INFO [RS:0;f2b92657890a:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,756 INFO [RS:0;f2b92657890a:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,756 INFO [RS:0;f2b92657890a:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,756 INFO [RS:0;f2b92657890a:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,756 INFO [RS:0;f2b92657890a:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,35473,1732473934123-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T18:45:35,771 WARN [f2b92657890a:45369 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T18:45:35,782 INFO [RS:1;f2b92657890a:38015 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T18:45:35,785 INFO [RS:1;f2b92657890a:38015 {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,38015,1732473934260-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,785 INFO [RS:0;f2b92657890a:35473 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T18:45:35,785 INFO [RS:1;f2b92657890a:38015 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,786 INFO [RS:1;f2b92657890a:38015 {}] regionserver.Replication(171): f2b92657890a,38015,1732473934260 started 2024-11-24T18:45:35,786 INFO [RS:0;f2b92657890a:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,35473,1732473934123-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,787 INFO [RS:0;f2b92657890a:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,788 INFO [RS:0;f2b92657890a:35473 {}] regionserver.Replication(171): f2b92657890a,35473,1732473934123 started 2024-11-24T18:45:35,810 INFO [RS:1;f2b92657890a:38015 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,811 INFO [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(1482): Serving as f2b92657890a,38015,1732473934260, RpcServer on f2b92657890a/172.17.0.2:38015, sessionid=0x1016e2a480d0002 2024-11-24T18:45:35,812 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T18:45:35,812 DEBUG [RS:1;f2b92657890a:38015 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f2b92657890a,38015,1732473934260 2024-11-24T18:45:35,812 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f2b92657890a,38015,1732473934260' 2024-11-24T18:45:35,812 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T18:45:35,814 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T18:45:35,815 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T18:45:35,815 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T18:45:35,815 DEBUG [RS:1;f2b92657890a:38015 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f2b92657890a,38015,1732473934260 2024-11-24T18:45:35,816 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f2b92657890a,38015,1732473934260' 2024-11-24T18:45:35,816 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T18:45:35,821 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T18:45:35,822 DEBUG [RS:1;f2b92657890a:38015 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T18:45:35,822 INFO [RS:1;f2b92657890a:38015 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T18:45:35,822 INFO [RS:1;f2b92657890a:38015 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T18:45:35,822 INFO [RS:0;f2b92657890a:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:35,823 INFO [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(1482): Serving as f2b92657890a,35473,1732473934123, RpcServer on f2b92657890a/172.17.0.2:35473, sessionid=0x1016e2a480d0001 2024-11-24T18:45:35,824 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T18:45:35,824 DEBUG [RS:0;f2b92657890a:35473 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f2b92657890a,35473,1732473934123 2024-11-24T18:45:35,824 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f2b92657890a,35473,1732473934123' 2024-11-24T18:45:35,824 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T18:45:35,825 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T18:45:35,826 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T18:45:35,826 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T18:45:35,826 DEBUG [RS:0;f2b92657890a:35473 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f2b92657890a,35473,1732473934123 2024-11-24T18:45:35,826 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f2b92657890a,35473,1732473934123' 2024-11-24T18:45:35,826 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T18:45:35,827 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T18:45:35,831 DEBUG [RS:0;f2b92657890a:35473 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T18:45:35,831 INFO [RS:0;f2b92657890a:35473 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T18:45:35,831 INFO [RS:0;f2b92657890a:35473 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T18:45:35,930 INFO [RS:1;f2b92657890a:38015 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T18:45:35,931 INFO [RS:1;f2b92657890a:38015 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f2b92657890a%2C38015%2C1732473934260, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,38015,1732473934260, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs, maxLogs=55 2024-11-24T18:45:35,933 INFO [RS:0;f2b92657890a:35473 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T18:45:35,934 INFO [RS:0;f2b92657890a:35473 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f2b92657890a%2C35473%2C1732473934123, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,35473,1732473934123, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs, maxLogs=55 2024-11-24T18:45:35,966 INFO [RS:1;f2b92657890a:38015 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,38015,1732473934260/f2b92657890a%2C38015%2C1732473934260.1732473935935 2024-11-24T18:45:35,967 INFO [RS:0;f2b92657890a:35473 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,35473,1732473934123/f2b92657890a%2C35473%2C1732473934123.1732473935938 2024-11-24T18:45:35,968 DEBUG [RS:0;f2b92657890a:35473 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T18:45:35,968 DEBUG [RS:1;f2b92657890a:38015 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T18:45:36,025 DEBUG [f2b92657890a:45369 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T18:45:36,038 DEBUG [f2b92657890a:45369 {}] balancer.BalancerClusterState(204): Hosts are {f2b92657890a=0} racks are {/default-rack=0} 2024-11-24T18:45:36,043 DEBUG [f2b92657890a:45369 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T18:45:36,043 DEBUG [f2b92657890a:45369 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T18:45:36,043 DEBUG [f2b92657890a:45369 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T18:45:36,043 DEBUG [f2b92657890a:45369 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T18:45:36,043 INFO [f2b92657890a:45369 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T18:45:36,043 INFO [f2b92657890a:45369 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T18:45:36,043 DEBUG [f2b92657890a:45369 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T18:45:36,051 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f2b92657890a,35473,1732473934123 2024-11-24T18:45:36,056 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f2b92657890a,35473,1732473934123, state=OPENING 2024-11-24T18:45:36,116 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T18:45:36,127 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:36,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:36,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:36,128 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T18:45:36,128 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T18:45:36,128 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T18:45:36,129 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T18:45:36,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f2b92657890a,35473,1732473934123}] 2024-11-24T18:45:36,310 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T18:45:36,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57641, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T18:45:36,320 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T18:45:36,321 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T18:45:36,321 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T18:45:36,323 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f2b92657890a%2C35473%2C1732473934123.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,35473,1732473934123, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs, maxLogs=55 2024-11-24T18:45:36,342 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,35473,1732473934123/f2b92657890a%2C35473%2C1732473934123.meta.1732473936324.meta 2024-11-24T18:45:36,343 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T18:45:36,343 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T18:45:36,345 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T18:45:36,348 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T18:45:36,353 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T18:45:36,359 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T18:45:36,360 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:36,360 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T18:45:36,360 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T18:45:36,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T18:45:36,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T18:45:36,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:36,366 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:36,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T18:45:36,368 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T18:45:36,368 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:36,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:36,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T18:45:36,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T18:45:36,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:36,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:36,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T18:45:36,372 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T18:45:36,372 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:36,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:36,373 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T18:45:36,375 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740 2024-11-24T18:45:36,376 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740 2024-11-24T18:45:36,377 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T18:45:36,377 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T18:45:36,378 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T18:45:36,380 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T18:45:36,381 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66824755, jitterRate=-0.004233554005622864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T18:45:36,381 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T18:45:36,383 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732473936361Writing region info on filesystem at 1732473936361Initializing all the Stores at 1732473936362 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473936362Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473936362Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473936362Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473936362Cleaning up temporary data from old regions at 1732473936377 (+15 ms)Running coprocessor post-open hooks at 1732473936382 (+5 ms)Region opened successfully at 1732473936383 (+1 ms) 2024-11-24T18:45:36,391 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732473936302 2024-11-24T18:45:36,407 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T18:45:36,408 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T18:45:36,409 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f2b92657890a,35473,1732473934123 2024-11-24T18:45:36,411 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f2b92657890a,35473,1732473934123, state=OPEN 2024-11-24T18:45:36,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T18:45:36,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T18:45:36,422 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T18:45:36,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T18:45:36,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T18:45:36,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T18:45:36,423 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f2b92657890a,35473,1732473934123 2024-11-24T18:45:36,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T18:45:36,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f2b92657890a,35473,1732473934123 in 292 msec 2024-11-24T18:45:36,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T18:45:36,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 825 msec 2024-11-24T18:45:36,436 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T18:45:36,436 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T18:45:36,476 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T18:45:36,478 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f2b92657890a,35473,1732473934123, seqNum=-1] 2024-11-24T18:45:36,510 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T18:45:36,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55873, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T18:45:36,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1570 sec 2024-11-24T18:45:36,547 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732473936547, completionTime=-1 2024-11-24T18:45:36,551 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T18:45:36,552 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T18:45:36,587 INFO [master/f2b92657890a:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T18:45:36,588 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732473996587 2024-11-24T18:45:36,588 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732474056588 2024-11-24T18:45:36,588 INFO [master/f2b92657890a:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 36 msec 2024-11-24T18:45:36,591 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T18:45:36,599 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,45369,1732473933194-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:36,600 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,45369,1732473933194-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:36,600 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,45369,1732473933194-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:36,602 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f2b92657890a:45369, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:36,602 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:36,603 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:36,610 DEBUG [master/f2b92657890a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T18:45:36,639 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.236sec 2024-11-24T18:45:36,642 INFO [master/f2b92657890a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T18:45:36,644 INFO [master/f2b92657890a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T18:45:36,645 INFO [master/f2b92657890a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T18:45:36,646 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T18:45:36,646 INFO [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T18:45:36,647 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,45369,1732473933194-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T18:45:36,648 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,45369,1732473933194-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T18:45:36,649 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T18:45:36,651 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T18:45:36,651 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,45369,1732473933194-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:36,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8eba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T18:45:36,718 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T18:45:36,718 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T18:45:36,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f2b92657890a,45369,-1 for getting cluster id 2024-11-24T18:45:36,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T18:45:36,730 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f3e3649e-c61b-41b2-9411-3090f0cf9945' 2024-11-24T18:45:36,732 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T18:45:36,732 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f3e3649e-c61b-41b2-9411-3090f0cf9945" 2024-11-24T18:45:36,733 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23449c56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T18:45:36,733 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f2b92657890a,45369,-1] 2024-11-24T18:45:36,735 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T18:45:36,737 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:36,738 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T18:45:36,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fe5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T18:45:36,741 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T18:45:36,748 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f2b92657890a,35473,1732473934123, seqNum=-1] 2024-11-24T18:45:36,749 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T18:45:36,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T18:45:36,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f2b92657890a,45369,1732473933194 2024-11-24T18:45:36,784 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T18:45:36,794 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f2b92657890a,45369,1732473933194 2024-11-24T18:45:36,797 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2092d9da 2024-11-24T18:45:36,799 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T18:45:36,807 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T18:45:36,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45369 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732473936780', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T18:45:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45369 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732473936780 2024-11-24T18:45:36,827 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732473936780 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T18:45:36,830 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45369 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732473936780" procId is: 4 2024-11-24T18:45:36,830 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:36,833 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732473936780 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T18:45:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45369 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T18:45:36,896 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732473936780-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => abe1f310d36181dce9ddae59f10597d3, NAME => 'testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732473936780', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9 2024-11-24T18:45:36,905 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732473936780-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:36,905 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732473936780-pool-0 {}] regionserver.HRegion(1722): Closing abe1f310d36181dce9ddae59f10597d3, disabling compactions & flushes 2024-11-24T18:45:36,905 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732473936780-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:36,906 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732473936780-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:36,906 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732473936780-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. after waiting 0 ms 2024-11-24T18:45:36,906 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732473936780-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:36,906 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732473936780-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:36,906 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732473936780-pool-0 {}] regionserver.HRegion(1676): Region close journal for abe1f310d36181dce9ddae59f10597d3: Waiting for close lock at 1732473936905Disabling compacts and flushes for region at 1732473936905Disabling writes for close at 1732473936906 (+1 ms)Writing region close event to WAL at 1732473936906Closed at 1732473936906 2024-11-24T18:45:36,908 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732473936780 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T18:45:36,912 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732473936908"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732473936908"}]},"ts":"1732473936908"} 2024-11-24T18:45:36,917 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T18:45:36,918 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732473936780 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T18:45:36,921 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732473936780","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732473936919"}]},"ts":"1732473936919"} 2024-11-24T18:45:36,925 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732473936780, state=ENABLING in hbase:meta 2024-11-24T18:45:36,925 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f2b92657890a=0} racks are {/default-rack=0} 2024-11-24T18:45:36,926 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T18:45:36,926 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T18:45:36,926 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T18:45:36,926 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T18:45:36,926 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T18:45:36,926 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T18:45:36,926 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T18:45:36,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732473936780, region=abe1f310d36181dce9ddae59f10597d3, ASSIGN}] 2024-11-24T18:45:36,929 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732473936780, region=abe1f310d36181dce9ddae59f10597d3, ASSIGN 2024-11-24T18:45:36,930 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732473936780, region=abe1f310d36181dce9ddae59f10597d3, ASSIGN; state=OFFLINE, location=f2b92657890a,35473,1732473934123; forceNewPlan=false, retain=false 2024-11-24T18:45:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45369 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T18:45:37,083 INFO [f2b92657890a:45369 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T18:45:37,084 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=abe1f310d36181dce9ddae59f10597d3, regionState=OPENING, regionLocation=f2b92657890a,35473,1732473934123 2024-11-24T18:45:37,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732473936780, region=abe1f310d36181dce9ddae59f10597d3, ASSIGN because future has completed 2024-11-24T18:45:37,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure abe1f310d36181dce9ddae59f10597d3, server=f2b92657890a,35473,1732473934123}] 2024-11-24T18:45:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45369 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T18:45:37,249 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:37,249 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => abe1f310d36181dce9ddae59f10597d3, NAME => 'testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3.', STARTKEY => '', ENDKEY => ''} 2024-11-24T18:45:37,250 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732473936780 abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:37,250 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:37,250 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:37,250 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:37,251 INFO [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:37,257 INFO [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abe1f310d36181dce9ddae59f10597d3 columnFamilyName family 2024-11-24T18:45:37,258 DEBUG [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:37,258 INFO [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] regionserver.HStore(327): Store=abe1f310d36181dce9ddae59f10597d3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T18:45:37,259 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:37,260 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:37,260 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:37,261 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:37,261 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:37,262 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:37,272 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T18:45:37,273 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened abe1f310d36181dce9ddae59f10597d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64830448, jitterRate=-0.0339510440826416}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T18:45:37,273 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:37,275 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for abe1f310d36181dce9ddae59f10597d3: Running coprocessor pre-open hook at 1732473937250Writing region info on filesystem at 1732473937251 (+1 ms)Initializing all the Stores at 1732473937251Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473937251Cleaning up temporary data from old regions at 1732473937261 (+10 ms)Running coprocessor post-open hooks at 1732473937273 (+12 ms)Region opened successfully at 1732473937274 (+1 ms) 2024-11-24T18:45:37,276 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3., pid=6, masterSystemTime=1732473937245 2024-11-24T18:45:37,281 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:37,282 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:37,282 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=abe1f310d36181dce9ddae59f10597d3, regionState=OPEN, openSeqNum=2, regionLocation=f2b92657890a,35473,1732473934123 2024-11-24T18:45:37,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure abe1f310d36181dce9ddae59f10597d3, server=f2b92657890a,35473,1732473934123 because future has completed 2024-11-24T18:45:37,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T18:45:37,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure abe1f310d36181dce9ddae59f10597d3, server=f2b92657890a,35473,1732473934123 in 200 msec 2024-11-24T18:45:37,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T18:45:37,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732473936780, region=abe1f310d36181dce9ddae59f10597d3, ASSIGN in 366 msec 2024-11-24T18:45:37,296 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732473936780 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T18:45:37,296 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732473936780","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732473937296"}]},"ts":"1732473937296"} 2024-11-24T18:45:37,300 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732473936780, state=ENABLED in hbase:meta 2024-11-24T18:45:37,301 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732473936780 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T18:45:37,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732473936780 in 482 msec 2024-11-24T18:45:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45369 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T18:45:37,469 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732473936780 get assigned. Timeout = 60000ms 2024-11-24T18:45:37,469 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732473936780 completed 2024-11-24T18:45:37,474 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T18:45:37,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732473936780 assigned to meta. Checking AM states. 2024-11-24T18:45:37,482 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T18:45:37,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732473936780 assigned. 2024-11-24T18:45:37,492 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732473936780', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3., hostname=f2b92657890a,35473,1732473934123, seqNum=2] 2024-11-24T18:45:37,504 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing abe1f310d36181dce9ddae59f10597d3 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T18:45:37,529 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 26d9b0667dbb4989ba859d191364467b 2024-11-24T18:45:37,575 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/.tmp/family/26d9b0667dbb4989ba859d191364467b is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T18:45:37,584 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/.tmp/family/26d9b0667dbb4989ba859d191364467b 2024-11-24T18:45:37,630 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/.tmp/family/26d9b0667dbb4989ba859d191364467b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/family/26d9b0667dbb4989ba859d191364467b 2024-11-24T18:45:37,632 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/family/26d9b0667dbb4989ba859d191364467b, delay=970 ms 2024-11-24T18:45:37,634 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/family/26d9b0667dbb4989ba859d191364467b, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T18:45:37,645 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for abe1f310d36181dce9ddae59f10597d3 in 137ms, sequenceid=6, compaction requested=false 2024-11-24T18:45:37,646 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732473936780' 2024-11-24T18:45:37,650 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for abe1f310d36181dce9ddae59f10597d3: 2024-11-24T18:45:38,606 DEBUG [hfile-prefetch-1732473937632 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 26d9b0667dbb4989ba859d191364467b_0. 2024-11-24T18:45:38,617 DEBUG [hfile-prefetch-1732473937632 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 26d9b0667dbb4989ba859d191364467b_117. 2024-11-24T18:45:38,618 DEBUG [hfile-prefetch-1732473937632 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/family/26d9b0667dbb4989ba859d191364467b, with total blocks 2, and data blocks 1 2024-11-24T18:45:38,618 DEBUG [hfile-prefetch-1732473937632 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T18:45:38,619 DEBUG [hfile-prefetch-1732473937632 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/family/26d9b0667dbb4989ba859d191364467b, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T18:45:38,619 DEBUG [hfile-prefetch-1732473937632 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/family/26d9b0667dbb4989ba859d191364467b. Will sleep 100ms and try the verification again. 2024-11-24T18:45:38,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45369 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=abe1f310d36181dce9ddae59f10597d3, source=f2b92657890a,35473,1732473934123, destination=f2b92657890a,38015,1732473934260, warming up region on f2b92657890a,38015,1732473934260 2024-11-24T18:45:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45369 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T18:45:38,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45369 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=abe1f310d36181dce9ddae59f10597d3, source=f2b92657890a,35473,1732473934123, destination=f2b92657890a,38015,1732473934260, running balancer 2024-11-24T18:45:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45369 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732473936780, region=abe1f310d36181dce9ddae59f10597d3, REOPEN/MOVE 2024-11-24T18:45:38,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49769, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T18:45:38,667 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732473936780, region=abe1f310d36181dce9ddae59f10597d3, REOPEN/MOVE 2024-11-24T18:45:38,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38015 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:38,669 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=abe1f310d36181dce9ddae59f10597d3, regionState=CLOSING, regionLocation=f2b92657890a,35473,1732473934123 2024-11-24T18:45:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38015 {}] regionserver.HRegion(7855): Warmup {ENCODED => abe1f310d36181dce9ddae59f10597d3, NAME => 'testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3.', STARTKEY => '', ENDKEY => ''} 2024-11-24T18:45:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38015 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:38,669 INFO [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:38,670 INFO [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abe1f310d36181dce9ddae59f10597d3 columnFamilyName family 2024-11-24T18:45:38,670 DEBUG [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:38,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732473936780, region=abe1f310d36181dce9ddae59f10597d3, REOPEN/MOVE because future has completed 2024-11-24T18:45:38,672 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-24T18:45:38,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure abe1f310d36181dce9ddae59f10597d3, server=f2b92657890a,35473,1732473934123}] 2024-11-24T18:45:38,674 DEBUG [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/family/26d9b0667dbb4989ba859d191364467b 2024-11-24T18:45:38,674 INFO [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] regionserver.HStore(327): Store=abe1f310d36181dce9ddae59f10597d3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T18:45:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38015 {}] regionserver.HRegion(1722): Closing abe1f310d36181dce9ddae59f10597d3, disabling compactions & flushes 2024-11-24T18:45:38,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38015 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38015 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38015 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. after waiting 0 ms 2024-11-24T18:45:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38015 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:38,676 DEBUG [StoreFileCloser-abe1f310d36181dce9ddae59f10597d3-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/family/26d9b0667dbb4989ba859d191364467b 2024-11-24T18:45:38,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38015 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38015 {}] regionserver.HRegion(1676): Region close journal for abe1f310d36181dce9ddae59f10597d3: Waiting for close lock at 1732473938674Disabling compacts and flushes for region at 1732473938674Disabling writes for close at 1732473938675 (+1 ms)Writing region close event to WAL at 1732473938676 (+1 ms)Closed at 1732473938676 2024-11-24T18:45:38,678 WARN [hfile-prefetch-1732473937632 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/family/26d9b0667dbb4989ba859d191364467b, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-24T18:45:38,678 DEBUG [hfile-prefetch-1732473937632 {}] hfile.PrefetchExecutor(121): Prefetch completed for 26d9b0667dbb4989ba859d191364467b 2024-11-24T18:45:38,692 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T18:45:38,692 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T18:45:38,693 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T18:45:38,693 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T18:45:38,709 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T18:45:38,709 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732473938693 2024-11-24T18:45:38,709 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T18:45:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45369 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T18:45:38,833 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:38,833 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-24T18:45:38,834 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing abe1f310d36181dce9ddae59f10597d3, disabling compactions & flushes 2024-11-24T18:45:38,834 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:38,834 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:38,835 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. after waiting 0 ms 2024-11-24T18:45:38,835 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:38,839 DEBUG [StoreFileCloser-abe1f310d36181dce9ddae59f10597d3-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 26d9b0667dbb4989ba859d191364467b, starting offset: 0, end offset: 9223372036854775807 2024-11-24T18:45:38,841 DEBUG [StoreFileCloser-abe1f310d36181dce9ddae59f10597d3-family-1 {}] bucket.BucketCache(856): removed key 26d9b0667dbb4989ba859d191364467b_0 from back map with offset lock 2094080 in the evict process 2024-11-24T18:45:38,846 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T18:45:38,848 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:38,848 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for abe1f310d36181dce9ddae59f10597d3: Waiting for close lock at 1732473938834Running coprocessor pre-close hooks at 1732473938834Disabling compacts and flushes for region at 1732473938834Disabling writes for close at 1732473938835 (+1 ms)Writing region close event to WAL at 1732473938842 (+7 ms)Running coprocessor post-close hooks at 1732473938847 (+5 ms)Closed at 1732473938848 (+1 ms) 2024-11-24T18:45:38,848 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding abe1f310d36181dce9ddae59f10597d3 move to f2b92657890a,38015,1732473934260 record at close sequenceid=6 2024-11-24T18:45:38,851 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=abe1f310d36181dce9ddae59f10597d3, regionState=CLOSED 2024-11-24T18:45:38,853 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:38,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure abe1f310d36181dce9ddae59f10597d3, server=f2b92657890a,35473,1732473934123 because future has completed 2024-11-24T18:45:38,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T18:45:38,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure abe1f310d36181dce9ddae59f10597d3, server=f2b92657890a,35473,1732473934123 in 183 msec 2024-11-24T18:45:38,858 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732473936780, region=abe1f310d36181dce9ddae59f10597d3, REOPEN/MOVE; state=CLOSED, location=f2b92657890a,38015,1732473934260; forceNewPlan=false, retain=false 2024-11-24T18:45:38,909 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T18:45:38,909 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T18:45:38,909 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T18:45:38,909 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T18:45:38,912 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T18:45:38,912 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732473938909 2024-11-24T18:45:38,912 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T18:45:39,009 INFO [f2b92657890a:45369 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T18:45:39,010 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=abe1f310d36181dce9ddae59f10597d3, regionState=OPENING, regionLocation=f2b92657890a,38015,1732473934260 2024-11-24T18:45:39,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732473936780, region=abe1f310d36181dce9ddae59f10597d3, REOPEN/MOVE because future has completed 2024-11-24T18:45:39,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure abe1f310d36181dce9ddae59f10597d3, server=f2b92657890a,38015,1732473934260}] 2024-11-24T18:45:39,176 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:39,177 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => abe1f310d36181dce9ddae59f10597d3, NAME => 'testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3.', STARTKEY => '', ENDKEY => ''} 2024-11-24T18:45:39,177 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732473936780 abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,177 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:39,178 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,178 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,179 INFO [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,180 INFO [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abe1f310d36181dce9ddae59f10597d3 columnFamilyName family 2024-11-24T18:45:39,180 DEBUG [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:39,182 DEBUG [StoreFileOpener-abe1f310d36181dce9ddae59f10597d3-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/family/26d9b0667dbb4989ba859d191364467b, delay=938 ms 2024-11-24T18:45:39,184 DEBUG [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/family/26d9b0667dbb4989ba859d191364467b 2024-11-24T18:45:39,184 INFO [StoreOpener-abe1f310d36181dce9ddae59f10597d3-1 {}] regionserver.HStore(327): Store=abe1f310d36181dce9ddae59f10597d3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T18:45:39,184 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,185 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,186 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,186 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,186 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,190 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,191 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened abe1f310d36181dce9ddae59f10597d3; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66266628, jitterRate=-0.012550294399261475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T18:45:39,191 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,192 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for abe1f310d36181dce9ddae59f10597d3: Running coprocessor pre-open hook at 1732473939178Writing region info on filesystem at 1732473939178Initializing all the Stores at 1732473939178Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473939178Cleaning up temporary data from old regions at 1732473939186 (+8 ms)Running coprocessor post-open hooks at 1732473939191 (+5 ms)Region opened successfully at 1732473939192 (+1 ms) 2024-11-24T18:45:39,193 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3., pid=9, masterSystemTime=1732473939174 2024-11-24T18:45:39,196 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:39,196 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:39,196 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=abe1f310d36181dce9ddae59f10597d3, regionState=OPEN, openSeqNum=10, regionLocation=f2b92657890a,38015,1732473934260 2024-11-24T18:45:39,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure abe1f310d36181dce9ddae59f10597d3, server=f2b92657890a,38015,1732473934260 because future has completed 2024-11-24T18:45:39,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T18:45:39,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure abe1f310d36181dce9ddae59f10597d3, server=f2b92657890a,38015,1732473934260 in 183 msec 2024-11-24T18:45:39,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732473936780, region=abe1f310d36181dce9ddae59f10597d3, REOPEN/MOVE in 538 msec 2024-11-24T18:45:39,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T18:45:39,220 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T18:45:39,220 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T18:45:39,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:39,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:39,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T18:45:39,228 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T18:45:39,228 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-24T18:45:39,229 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f2b92657890a,45369,1732473933194 2024-11-24T18:45:39,253 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T18:45:39,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T18:45:39,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T18:45:39,253 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:39,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:39,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:39,253 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T18:45:39,254 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T18:45:39,254 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T18:45:39,254 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T18:45:39,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:39,254 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T18:45:39,254 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T18:45:39,255 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f2b92657890a,35473,1732473934123' ***** 2024-11-24T18:45:39,255 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T18:45:39,255 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f2b92657890a,38015,1732473934260' ***** 2024-11-24T18:45:39,255 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T18:45:39,255 INFO [RS:0;f2b92657890a:35473 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T18:45:39,255 INFO [RS:1;f2b92657890a:38015 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T18:45:39,256 INFO [RS:1;f2b92657890a:38015 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/bucket.persistence 2024-11-24T18:45:39,256 INFO [RS:0;f2b92657890a:35473 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/bucket.persistence 2024-11-24T18:45:39,256 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T18:45:39,256 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T18:45:39,256 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T18:45:39,256 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T18:45:39,256 INFO [RS:0;f2b92657890a:35473 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T18:45:39,256 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T18:45:39,256 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T18:45:39,256 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T18:45:39,256 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T18:45:39,256 INFO [RS:1;f2b92657890a:38015 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T18:45:39,257 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T18:45:39,260 INFO [regionserver/f2b92657890a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T18:45:39,266 DEBUG [RS:0;f2b92657890a:35473 {}] bucket.BucketCache(1487): Thread RS:0;f2b92657890a:35473 started persisting bucket cache to file 2024-11-24T18:45:39,266 DEBUG [RS:0;f2b92657890a:35473 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T18:45:39,266 DEBUG [RS:0;f2b92657890a:35473 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T18:45:39,267 DEBUG [RS:1;f2b92657890a:38015 {}] bucket.BucketCache(1487): Thread RS:1;f2b92657890a:38015 started persisting bucket cache to file 2024-11-24T18:45:39,267 DEBUG [RS:1;f2b92657890a:38015 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T18:45:39,267 DEBUG [RS:1;f2b92657890a:38015 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T18:45:39,269 DEBUG [RS:0;f2b92657890a:35473 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T18:45:39,270 DEBUG [RS:0;f2b92657890a:35473 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732473939266 2024-11-24T18:45:39,270 DEBUG [RS:1;f2b92657890a:38015 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T18:45:39,270 DEBUG [RS:0;f2b92657890a:35473 {}] bucket.BucketCache(1509): Thread RS:0;f2b92657890a:35473 finished persisting bucket cache to file, renaming 2024-11-24T18:45:39,270 DEBUG [RS:1;f2b92657890a:38015 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732473939267 2024-11-24T18:45:39,270 DEBUG [RS:1;f2b92657890a:38015 {}] bucket.BucketCache(1509): Thread RS:1;f2b92657890a:38015 finished persisting bucket cache to file, renaming 2024-11-24T18:45:39,270 INFO [RS:1;f2b92657890a:38015 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T18:45:39,270 INFO [RS:0;f2b92657890a:35473 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T18:45:39,270 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T18:45:39,270 INFO [RS:0;f2b92657890a:35473 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T18:45:39,270 INFO [RS:1;f2b92657890a:38015 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T18:45:39,270 INFO [RS:0;f2b92657890a:35473 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T18:45:39,270 INFO [RS:1;f2b92657890a:38015 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T18:45:39,271 INFO [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(959): stopping server f2b92657890a,35473,1732473934123 2024-11-24T18:45:39,271 INFO [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(3091): Received CLOSE for abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,271 INFO [RS:0;f2b92657890a:35473 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T18:45:39,271 INFO [RS:0;f2b92657890a:35473 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f2b92657890a:35473. 2024-11-24T18:45:39,271 DEBUG [RS:0;f2b92657890a:35473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T18:45:39,271 DEBUG [RS:0;f2b92657890a:35473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:39,271 INFO [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(959): stopping server f2b92657890a,38015,1732473934260 2024-11-24T18:45:39,271 INFO [RS:1;f2b92657890a:38015 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T18:45:39,271 INFO [RS:0;f2b92657890a:35473 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T18:45:39,271 INFO [RS:1;f2b92657890a:38015 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f2b92657890a:38015. 2024-11-24T18:45:39,271 INFO [RS:0;f2b92657890a:35473 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T18:45:39,271 INFO [RS:0;f2b92657890a:35473 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T18:45:39,271 DEBUG [RS:1;f2b92657890a:38015 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T18:45:39,271 INFO [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T18:45:39,271 DEBUG [RS:1;f2b92657890a:38015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:39,272 INFO [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T18:45:39,272 DEBUG [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(1325): Online Regions={abe1f310d36181dce9ddae59f10597d3=testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3.} 2024-11-24T18:45:39,272 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T18:45:39,272 DEBUG [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(1351): Waiting on abe1f310d36181dce9ddae59f10597d3 2024-11-24T18:45:39,272 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing abe1f310d36181dce9ddae59f10597d3, disabling compactions & flushes 2024-11-24T18:45:39,272 INFO [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T18:45:39,272 DEBUG [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T18:45:39,272 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:39,272 DEBUG [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T18:45:39,272 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:39,272 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T18:45:39,272 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. after waiting 0 ms 2024-11-24T18:45:39,272 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T18:45:39,272 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:39,272 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T18:45:39,273 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T18:45:39,273 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T18:45:39,273 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T18:45:39,273 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-24T18:45:39,273 DEBUG [StoreFileCloser-abe1f310d36181dce9ddae59f10597d3-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/family/26d9b0667dbb4989ba859d191364467b 2024-11-24T18:45:39,279 INFO [regionserver/f2b92657890a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T18:45:39,280 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnRegionMove1732473936780/abe1f310d36181dce9ddae59f10597d3/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T18:45:39,281 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:39,281 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for abe1f310d36181dce9ddae59f10597d3: Waiting for close lock at 1732473939271Running coprocessor pre-close hooks at 1732473939272 (+1 ms)Disabling compacts and flushes for region at 1732473939272Disabling writes for close at 1732473939272Writing region close event to WAL at 1732473939273 (+1 ms)Running coprocessor post-close hooks at 1732473939280 (+7 ms)Closed at 1732473939280 2024-11-24T18:45:39,281 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3. 2024-11-24T18:45:39,283 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e6b8d50194d84990a0569c6e2e96a8b9 2024-11-24T18:45:39,302 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/info/e6b8d50194d84990a0569c6e2e96a8b9 is 205, key is testBlockEvictionOnRegionMove1732473936780,,1732473936809.abe1f310d36181dce9ddae59f10597d3./info:regioninfo/1732473939196/Put/seqid=0 2024-11-24T18:45:39,303 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/info/e6b8d50194d84990a0569c6e2e96a8b9 2024-11-24T18:45:39,314 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ac799bd975384a8d89197d852d6726af 2024-11-24T18:45:39,334 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/ns/ac799bd975384a8d89197d852d6726af is 43, key is default/ns:d/1732473936521/Put/seqid=0 2024-11-24T18:45:39,335 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/ns/ac799bd975384a8d89197d852d6726af 2024-11-24T18:45:39,346 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ad34523e44e94810b54a7db649956b2d 2024-11-24T18:45:39,358 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/table/ad34523e44e94810b54a7db649956b2d is 78, key is testBlockEvictionOnRegionMove1732473936780/table:state/1732473937296/Put/seqid=0 2024-11-24T18:45:39,359 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/table/ad34523e44e94810b54a7db649956b2d 2024-11-24T18:45:39,362 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/info/e6b8d50194d84990a0569c6e2e96a8b9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/info/e6b8d50194d84990a0569c6e2e96a8b9 2024-11-24T18:45:39,363 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/info/e6b8d50194d84990a0569c6e2e96a8b9, delay=986 ms 2024-11-24T18:45:39,365 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/info/e6b8d50194d84990a0569c6e2e96a8b9, entries=15, sequenceid=15, filesize=7.6 K 2024-11-24T18:45:39,366 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/ns/ac799bd975384a8d89197d852d6726af as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/ns/ac799bd975384a8d89197d852d6726af 2024-11-24T18:45:39,367 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/ns/ac799bd975384a8d89197d852d6726af, delay=940 ms 2024-11-24T18:45:39,369 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/ns/ac799bd975384a8d89197d852d6726af, entries=2, sequenceid=15, filesize=5.0 K 2024-11-24T18:45:39,370 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/table/ad34523e44e94810b54a7db649956b2d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/table/ad34523e44e94810b54a7db649956b2d 2024-11-24T18:45:39,371 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/table/ad34523e44e94810b54a7db649956b2d, delay=920 ms 2024-11-24T18:45:39,373 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/table/ad34523e44e94810b54a7db649956b2d, entries=2, sequenceid=15, filesize=5.3 K 2024-11-24T18:45:39,374 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=15, compaction requested=false 2024-11-24T18:45:39,375 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T18:45:39,376 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/info/e6b8d50194d84990a0569c6e2e96a8b9 2024-11-24T18:45:39,380 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/ns/ac799bd975384a8d89197d852d6726af 2024-11-24T18:45:39,383 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/table/ad34523e44e94810b54a7db649956b2d 2024-11-24T18:45:39,387 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-24T18:45:39,388 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T18:45:39,389 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T18:45:39,389 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732473939272Running coprocessor pre-close hooks at 1732473939272Disabling compacts and flushes for region at 1732473939272Disabling writes for close at 1732473939273 (+1 ms)Obtaining lock to block concurrent updates at 1732473939273Preparing flush snapshotting stores in 1588230740 at 1732473939273Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732473939273Flushing stores of hbase:meta,,1.1588230740 at 1732473939274 (+1 ms)Flushing 1588230740/info: creating writer at 1732473939274Flushing 1588230740/info: appending metadata at 1732473939301 (+27 ms)Flushing 1588230740/info: closing flushed file at 1732473939301Flushing 1588230740/ns: creating writer at 1732473939307 (+6 ms)Flushing 1588230740/ns: appending metadata at 1732473939332 (+25 ms)Flushing 1588230740/ns: closing flushed file at 1732473939332Flushing 1588230740/table: creating writer at 1732473939338 (+6 ms)Flushing 1588230740/table: appending metadata at 1732473939358 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732473939358Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3076ddc7: reopening flushed file at 1732473939362 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73104afe: reopening flushed file at 1732473939365 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@670edb80: reopening flushed file at 1732473939369 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=15, compaction requested=false at 1732473939375 (+6 ms)Writing region close event to WAL at 1732473939383 (+8 ms)Running coprocessor post-close hooks at 1732473939388 (+5 ms)Closed at 1732473939389 (+1 ms) 2024-11-24T18:45:39,389 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T18:45:39,473 INFO [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(976): stopping server f2b92657890a,35473,1732473934123; all regions closed. 2024-11-24T18:45:39,473 INFO [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(976): stopping server f2b92657890a,38015,1732473934260; all regions closed. 2024-11-24T18:45:39,478 DEBUG [RS:1;f2b92657890a:38015 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs 2024-11-24T18:45:39,478 DEBUG [RS:0;f2b92657890a:35473 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs 2024-11-24T18:45:39,479 INFO [RS:1;f2b92657890a:38015 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f2b92657890a%2C38015%2C1732473934260:(num 1732473935935) 2024-11-24T18:45:39,479 DEBUG [RS:1;f2b92657890a:38015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:39,479 INFO [RS:0;f2b92657890a:35473 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f2b92657890a%2C35473%2C1732473934123.meta:.meta(num 1732473936324) 2024-11-24T18:45:39,479 INFO [RS:1;f2b92657890a:38015 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T18:45:39,479 INFO [RS:1;f2b92657890a:38015 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T18:45:39,479 INFO [RS:1;f2b92657890a:38015 {}] hbase.ChoreService(370): Chore service for: regionserver/f2b92657890a:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T18:45:39,480 INFO [RS:1;f2b92657890a:38015 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T18:45:39,480 INFO [RS:1;f2b92657890a:38015 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T18:45:39,480 INFO [RS:1;f2b92657890a:38015 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T18:45:39,480 INFO [RS:1;f2b92657890a:38015 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T18:45:39,480 INFO [RS:1;f2b92657890a:38015 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38015 2024-11-24T18:45:39,482 INFO [regionserver/f2b92657890a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T18:45:39,484 DEBUG [RS:0;f2b92657890a:35473 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs 2024-11-24T18:45:39,484 INFO [RS:0;f2b92657890a:35473 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f2b92657890a%2C35473%2C1732473934123:(num 1732473935938) 2024-11-24T18:45:39,484 DEBUG [RS:0;f2b92657890a:35473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:39,484 INFO [RS:0;f2b92657890a:35473 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T18:45:39,484 INFO [RS:0;f2b92657890a:35473 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T18:45:39,485 INFO [RS:0;f2b92657890a:35473 {}] hbase.ChoreService(370): Chore service for: regionserver/f2b92657890a:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T18:45:39,486 INFO [RS:0;f2b92657890a:35473 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T18:45:39,487 INFO [regionserver/f2b92657890a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T18:45:39,487 INFO [RS:0;f2b92657890a:35473 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35473 2024-11-24T18:45:39,503 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f2b92657890a,38015,1732473934260 2024-11-24T18:45:39,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T18:45:39,503 INFO [RS:1;f2b92657890a:38015 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T18:45:39,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f2b92657890a,35473,1732473934123 2024-11-24T18:45:39,594 INFO [RS:0;f2b92657890a:35473 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T18:45:39,594 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f2b92657890a,38015,1732473934260] 2024-11-24T18:45:39,632 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f2b92657890a,38015,1732473934260 already deleted, retry=false 2024-11-24T18:45:39,632 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f2b92657890a,38015,1732473934260 expired; onlineServers=1 2024-11-24T18:45:39,632 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f2b92657890a,35473,1732473934123] 2024-11-24T18:45:39,642 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f2b92657890a,35473,1732473934123 already deleted, retry=false 2024-11-24T18:45:39,642 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f2b92657890a,35473,1732473934123 expired; onlineServers=0 2024-11-24T18:45:39,643 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f2b92657890a,45369,1732473933194' ***** 2024-11-24T18:45:39,643 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T18:45:39,643 INFO [M:0;f2b92657890a:45369 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T18:45:39,643 INFO [M:0;f2b92657890a:45369 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T18:45:39,643 DEBUG [M:0;f2b92657890a:45369 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T18:45:39,643 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T18:45:39,643 DEBUG [M:0;f2b92657890a:45369 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T18:45:39,643 DEBUG [master/f2b92657890a:0:becomeActiveMaster-HFileCleaner.large.0-1732473935498 {}] cleaner.HFileCleaner(306): Exit Thread[master/f2b92657890a:0:becomeActiveMaster-HFileCleaner.large.0-1732473935498,5,FailOnTimeoutGroup] 2024-11-24T18:45:39,643 DEBUG [master/f2b92657890a:0:becomeActiveMaster-HFileCleaner.small.0-1732473935499 {}] cleaner.HFileCleaner(306): Exit Thread[master/f2b92657890a:0:becomeActiveMaster-HFileCleaner.small.0-1732473935499,5,FailOnTimeoutGroup] 2024-11-24T18:45:39,644 INFO [M:0;f2b92657890a:45369 {}] hbase.ChoreService(370): Chore service for: master/f2b92657890a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T18:45:39,644 INFO [M:0;f2b92657890a:45369 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T18:45:39,644 DEBUG [M:0;f2b92657890a:45369 {}] master.HMaster(1795): Stopping service threads 2024-11-24T18:45:39,644 INFO [M:0;f2b92657890a:45369 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T18:45:39,644 INFO [M:0;f2b92657890a:45369 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T18:45:39,645 INFO [M:0;f2b92657890a:45369 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T18:45:39,645 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T18:45:39,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T18:45:39,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:39,653 DEBUG [M:0;f2b92657890a:45369 {}] zookeeper.ZKUtil(347): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T18:45:39,653 WARN [M:0;f2b92657890a:45369 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T18:45:39,654 INFO [M:0;f2b92657890a:45369 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/.lastflushedseqids 2024-11-24T18:45:39,662 INFO [M:0;f2b92657890a:45369 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T18:45:39,662 INFO [M:0;f2b92657890a:45369 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T18:45:39,663 DEBUG [M:0;f2b92657890a:45369 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T18:45:39,663 INFO [M:0;f2b92657890a:45369 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:39,663 DEBUG [M:0;f2b92657890a:45369 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:39,663 DEBUG [M:0;f2b92657890a:45369 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T18:45:39,663 DEBUG [M:0;f2b92657890a:45369 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:39,663 INFO [M:0;f2b92657890a:45369 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-24T18:45:39,674 DEBUG [M:0;f2b92657890a:45369 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6e6bda00cc574b408b30d5d2682a5794 2024-11-24T18:45:39,687 DEBUG [M:0;f2b92657890a:45369 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6e6bda00cc574b408b30d5d2682a5794 is 82, key is hbase:meta,,1/info:regioninfo/1732473936408/Put/seqid=0 2024-11-24T18:45:39,688 INFO [M:0;f2b92657890a:45369 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6e6bda00cc574b408b30d5d2682a5794 2024-11-24T18:45:39,698 DEBUG [M:0;f2b92657890a:45369 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c63e970e297c4091bd2c09174bac3bac 2024-11-24T18:45:39,719 DEBUG [M:0;f2b92657890a:45369 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c63e970e297c4091bd2c09174bac3bac is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732473937303/Put/seqid=0 2024-11-24T18:45:39,720 INFO [M:0;f2b92657890a:45369 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c63e970e297c4091bd2c09174bac3bac 2024-11-24T18:45:39,722 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:39,723 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38015-0x1016e2a480d0002, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:39,723 INFO [RS:1;f2b92657890a:38015 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T18:45:39,723 INFO [RS:1;f2b92657890a:38015 {}] regionserver.HRegionServer(1031): Exiting; stopping=f2b92657890a,38015,1732473934260; zookeeper connection closed. 2024-11-24T18:45:39,725 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2ea5f285 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2ea5f285 2024-11-24T18:45:39,732 INFO [RS:0;f2b92657890a:35473 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T18:45:39,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:39,732 INFO [RS:0;f2b92657890a:35473 {}] regionserver.HRegionServer(1031): Exiting; stopping=f2b92657890a,35473,1732473934123; zookeeper connection closed. 2024-11-24T18:45:39,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1016e2a480d0001, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:39,732 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@51074cff {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@51074cff 2024-11-24T18:45:39,733 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-24T18:45:39,734 DEBUG [M:0;f2b92657890a:45369 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e8f1dcc4bd5b4095a4bec72b2f477382 2024-11-24T18:45:39,749 DEBUG [M:0;f2b92657890a:45369 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e8f1dcc4bd5b4095a4bec72b2f477382 is 69, key is f2b92657890a,35473,1732473934123/rs:state/1732473935595/Put/seqid=0 2024-11-24T18:45:39,750 INFO [M:0;f2b92657890a:45369 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e8f1dcc4bd5b4095a4bec72b2f477382 2024-11-24T18:45:39,753 DEBUG [M:0;f2b92657890a:45369 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6e6bda00cc574b408b30d5d2682a5794 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6e6bda00cc574b408b30d5d2682a5794 2024-11-24T18:45:39,755 INFO [M:0;f2b92657890a:45369 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6e6bda00cc574b408b30d5d2682a5794, entries=8, sequenceid=84, filesize=5.5 K 2024-11-24T18:45:39,756 DEBUG [M:0;f2b92657890a:45369 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c63e970e297c4091bd2c09174bac3bac as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c63e970e297c4091bd2c09174bac3bac 2024-11-24T18:45:39,759 INFO [M:0;f2b92657890a:45369 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c63e970e297c4091bd2c09174bac3bac, entries=9, sequenceid=84, filesize=6.5 K 2024-11-24T18:45:39,759 DEBUG [M:0;f2b92657890a:45369 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e8f1dcc4bd5b4095a4bec72b2f477382 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e8f1dcc4bd5b4095a4bec72b2f477382 2024-11-24T18:45:39,762 INFO [M:0;f2b92657890a:45369 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e8f1dcc4bd5b4095a4bec72b2f477382, entries=2, sequenceid=84, filesize=5.1 K 2024-11-24T18:45:39,763 INFO [M:0;f2b92657890a:45369 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=84, compaction requested=false 2024-11-24T18:45:39,765 INFO [M:0;f2b92657890a:45369 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:39,766 DEBUG [M:0;f2b92657890a:45369 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732473939663Disabling compacts and flushes for region at 1732473939663Disabling writes for close at 1732473939663Obtaining lock to block concurrent updates at 1732473939663Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732473939663Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732473939664 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732473939664Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732473939664Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732473939687 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732473939687Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732473939690 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732473939718 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732473939718Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732473939723 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732473939748 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732473939748Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2906ac9: reopening flushed file at 1732473939752 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f916aaa: reopening flushed file at 1732473939756 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b7a638d: reopening flushed file at 1732473939759 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=84, compaction requested=false at 1732473939763 (+4 ms)Writing region close event to WAL at 1732473939765 (+2 ms)Closed at 1732473939765 2024-11-24T18:45:39,771 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T18:45:39,771 INFO [M:0;f2b92657890a:45369 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T18:45:39,771 INFO [M:0;f2b92657890a:45369 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45369 2024-11-24T18:45:39,771 INFO [M:0;f2b92657890a:45369 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T18:45:39,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:39,879 INFO [M:0;f2b92657890a:45369 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T18:45:39,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45369-0x1016e2a480d0000, quorum=127.0.0.1:58182, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:39,885 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T18:45:39,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T18:45:39,916 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:39,927 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: master/f2b92657890a:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f2b92657890a:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732473939182 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/f2b92657890a:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f2b92657890a:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1732473939367 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732473939364 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732473937632 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=502 (was 511), ProcessCount=11 (was 11), AvailableMemoryMB=10384 (was 9344) - AvailableMemoryMB LEAK? - 2024-11-24T18:45:39,935 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=502, ProcessCount=11, AvailableMemoryMB=10381 2024-11-24T18:45:39,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/hadoop-log-dir so I do NOT create it in target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849 2024-11-24T18:45:39,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/hadoop-tmp-dir so I do NOT create it in target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849 2024-11-24T18:45:39,935 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:39,939 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/cluster_45359caa-ed9c-2215-f6b5-786f621746b5, deleteOnExit=true 2024-11-24T18:45:39,960 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/cluster_45359caa-ed9c-2215-f6b5-786f621746b5/zookeeper_0, clientPort=60450, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/cluster_45359caa-ed9c-2215-f6b5-786f621746b5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/cluster_45359caa-ed9c-2215-f6b5-786f621746b5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T18:45:39,963 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60450 2024-11-24T18:45:39,977 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9 with version=8 2024-11-24T18:45:39,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/hbase-staging 2024-11-24T18:45:39,980 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f2b92657890a:0 server-side Connection retries=45 2024-11-24T18:45:39,980 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:39,980 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:39,981 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T18:45:39,981 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:39,981 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T18:45:39,981 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T18:45:39,981 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T18:45:39,982 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43953 2024-11-24T18:45:39,983 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43953 connecting to ZooKeeper ensemble=127.0.0.1:60450 2024-11-24T18:45:40,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:439530x0, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T18:45:40,034 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43953-0x1016e2a63a20000 connected 2024-11-24T18:45:40,115 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:40,115 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:40,117 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T18:45:40,117 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9, hbase.cluster.distributed=false 2024-11-24T18:45:40,120 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T18:45:40,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43953 2024-11-24T18:45:40,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43953 2024-11-24T18:45:40,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43953 2024-11-24T18:45:40,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43953 2024-11-24T18:45:40,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43953 2024-11-24T18:45:40,147 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f2b92657890a:0 server-side Connection retries=45 2024-11-24T18:45:40,147 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:40,147 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:40,147 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T18:45:40,147 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:40,147 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T18:45:40,147 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T18:45:40,147 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T18:45:40,148 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40227 2024-11-24T18:45:40,150 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40227 connecting to ZooKeeper ensemble=127.0.0.1:60450 2024-11-24T18:45:40,150 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:40,151 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:40,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:402270x0, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T18:45:40,168 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40227-0x1016e2a63a20001 connected 2024-11-24T18:45:40,168 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T18:45:40,169 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T18:45:40,170 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.cache 2024-11-24T18:45:40,170 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T18:45:40,170 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T18:45:40,170 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T18:45:40,172 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T18:45:40,172 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T18:45:40,172 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T18:45:40,173 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T18:45:40,173 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T18:45:40,173 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.persistence is complete. 2024-11-24T18:45:40,174 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T18:45:40,176 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T18:45:40,178 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40227 2024-11-24T18:45:40,178 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40227 2024-11-24T18:45:40,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40227 2024-11-24T18:45:40,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40227 2024-11-24T18:45:40,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40227 2024-11-24T18:45:40,213 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f2b92657890a:0 server-side Connection retries=45 2024-11-24T18:45:40,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:40,214 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:40,214 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T18:45:40,214 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:40,214 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T18:45:40,214 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T18:45:40,214 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T18:45:40,215 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35197 2024-11-24T18:45:40,218 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35197 connecting to ZooKeeper ensemble=127.0.0.1:60450 2024-11-24T18:45:40,218 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:40,218 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:40,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:351970x0, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T18:45:40,231 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:351970x0, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T18:45:40,232 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35197-0x1016e2a63a20002 connected 2024-11-24T18:45:40,232 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T18:45:40,236 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.cache 2024-11-24T18:45:40,237 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T18:45:40,237 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T18:45:40,237 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T18:45:40,240 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T18:45:40,240 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T18:45:40,240 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T18:45:40,241 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T18:45:40,241 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T18:45:40,241 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.persistence is complete. 2024-11-24T18:45:40,243 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T18:45:40,245 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T18:45:40,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35197 2024-11-24T18:45:40,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35197 2024-11-24T18:45:40,252 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35197 2024-11-24T18:45:40,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35197 2024-11-24T18:45:40,256 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35197 2024-11-24T18:45:40,276 DEBUG [M:0;f2b92657890a:43953 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f2b92657890a:43953 2024-11-24T18:45:40,277 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f2b92657890a,43953,1732473939979 2024-11-24T18:45:40,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T18:45:40,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T18:45:40,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T18:45:40,284 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f2b92657890a,43953,1732473939979 2024-11-24T18:45:40,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T18:45:40,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T18:45:40,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,295 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T18:45:40,295 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f2b92657890a,43953,1732473939979 from backup master directory 2024-11-24T18:45:40,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T18:45:40,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f2b92657890a,43953,1732473939979 2024-11-24T18:45:40,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T18:45:40,305 WARN [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T18:45:40,305 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f2b92657890a,43953,1732473939979 2024-11-24T18:45:40,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T18:45:40,307 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/hbase.id] with ID: 0044d209-2696-4e94-8ad0-154f53912e1e 2024-11-24T18:45:40,307 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/.tmp/hbase.id 2024-11-24T18:45:40,312 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/hbase.id] 2024-11-24T18:45:40,361 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:40,361 INFO [master/f2b92657890a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T18:45:40,364 INFO [master/f2b92657890a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-24T18:45:40,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,384 INFO [master/f2b92657890a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T18:45:40,386 INFO [master/f2b92657890a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T18:45:40,386 INFO [master/f2b92657890a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T18:45:40,407 INFO [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store 2024-11-24T18:45:40,415 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:40,415 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T18:45:40,415 INFO [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:40,416 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:40,416 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T18:45:40,416 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:40,416 INFO [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:40,416 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732473940415Disabling compacts and flushes for region at 1732473940415Disabling writes for close at 1732473940416 (+1 ms)Writing region close event to WAL at 1732473940416Closed at 1732473940416 2024-11-24T18:45:40,416 WARN [master/f2b92657890a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/.initializing 2024-11-24T18:45:40,416 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/WALs/f2b92657890a,43953,1732473939979 2024-11-24T18:45:40,423 INFO [master/f2b92657890a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T18:45:40,427 INFO [master/f2b92657890a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f2b92657890a%2C43953%2C1732473939979, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/WALs/f2b92657890a,43953,1732473939979, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/oldWALs, maxLogs=10 2024-11-24T18:45:40,447 INFO [master/f2b92657890a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/WALs/f2b92657890a,43953,1732473939979/f2b92657890a%2C43953%2C1732473939979.1732473940427 2024-11-24T18:45:40,448 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T18:45:40,448 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T18:45:40,448 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:40,448 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:40,448 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:40,456 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:40,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T18:45:40,475 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:40,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:40,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:40,479 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T18:45:40,479 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:40,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T18:45:40,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:40,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T18:45:40,485 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:40,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T18:45:40,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:40,488 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T18:45:40,488 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:40,489 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T18:45:40,489 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:40,490 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:40,490 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:40,490 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:40,490 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:40,491 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T18:45:40,491 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T18:45:40,497 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T18:45:40,497 INFO [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65412849, jitterRate=-0.025272592902183533}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T18:45:40,499 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732473940448Initializing all the Stores at 1732473940449 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473940449Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473940456 (+7 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473940456Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473940456Cleaning up temporary data from old regions at 1732473940490 (+34 ms)Region opened successfully at 1732473940498 (+8 ms) 2024-11-24T18:45:40,499 INFO [master/f2b92657890a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T18:45:40,503 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63710ca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f2b92657890a/172.17.0.2:0 2024-11-24T18:45:40,505 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T18:45:40,505 INFO [master/f2b92657890a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T18:45:40,505 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T18:45:40,505 INFO [master/f2b92657890a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T18:45:40,505 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T18:45:40,506 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-24T18:45:40,506 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T18:45:40,508 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T18:45:40,509 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T18:45:40,514 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T18:45:40,515 INFO [master/f2b92657890a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T18:45:40,516 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T18:45:40,525 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T18:45:40,525 INFO [master/f2b92657890a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T18:45:40,527 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T18:45:40,535 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T18:45:40,537 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T18:45:40,546 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T18:45:40,549 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T18:45:40,557 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T18:45:40,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T18:45:40,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T18:45:40,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T18:45:40,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,570 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f2b92657890a,43953,1732473939979, sessionid=0x1016e2a63a20000, setting cluster-up flag (Was=false) 2024-11-24T18:45:40,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,620 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T18:45:40,622 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f2b92657890a,43953,1732473939979 2024-11-24T18:45:40,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:40,672 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T18:45:40,674 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f2b92657890a,43953,1732473939979 2024-11-24T18:45:40,674 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T18:45:40,676 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T18:45:40,676 INFO [master/f2b92657890a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T18:45:40,677 INFO [master/f2b92657890a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T18:45:40,677 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f2b92657890a,43953,1732473939979 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T18:45:40,679 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f2b92657890a:0, corePoolSize=5, maxPoolSize=5 2024-11-24T18:45:40,679 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f2b92657890a:0, corePoolSize=5, maxPoolSize=5 2024-11-24T18:45:40,679 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f2b92657890a:0, corePoolSize=5, maxPoolSize=5 2024-11-24T18:45:40,679 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f2b92657890a:0, corePoolSize=5, maxPoolSize=5 2024-11-24T18:45:40,679 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f2b92657890a:0, corePoolSize=10, maxPoolSize=10 2024-11-24T18:45:40,679 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,679 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f2b92657890a:0, corePoolSize=2, maxPoolSize=2 2024-11-24T18:45:40,680 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,683 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T18:45:40,683 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T18:45:40,684 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:40,684 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T18:45:40,688 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732473970688 2024-11-24T18:45:40,688 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T18:45:40,689 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T18:45:40,689 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T18:45:40,689 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T18:45:40,689 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T18:45:40,689 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T18:45:40,692 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,697 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T18:45:40,697 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T18:45:40,697 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T18:45:40,697 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T18:45:40,697 INFO [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T18:45:40,698 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f2b92657890a:0:becomeActiveMaster-HFileCleaner.large.0-1732473940697,5,FailOnTimeoutGroup] 2024-11-24T18:45:40,699 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f2b92657890a:0:becomeActiveMaster-HFileCleaner.small.0-1732473940698,5,FailOnTimeoutGroup] 2024-11-24T18:45:40,700 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,700 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T18:45:40,700 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,700 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,711 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T18:45:40,712 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9 2024-11-24T18:45:40,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:40,724 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T18:45:40,729 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T18:45:40,729 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:40,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:40,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T18:45:40,733 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T18:45:40,733 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:40,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:40,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T18:45:40,737 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T18:45:40,737 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:40,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:40,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T18:45:40,742 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T18:45:40,742 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:40,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:40,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T18:45:40,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740 2024-11-24T18:45:40,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740 2024-11-24T18:45:40,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T18:45:40,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T18:45:40,744 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T18:45:40,744 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T18:45:40,751 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T18:45:40,752 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63889583, jitterRate=-0.04797102510929108}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T18:45:40,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732473940723Initializing all the Stores at 1732473940723Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473940723Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473940724 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473940724Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473940724Cleaning up temporary data from old regions at 1732473940743 (+19 ms)Region opened successfully at 1732473940752 (+9 ms) 2024-11-24T18:45:40,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T18:45:40,753 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T18:45:40,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T18:45:40,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T18:45:40,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T18:45:40,753 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T18:45:40,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732473940752Disabling compacts and flushes for region at 1732473940752Disabling writes for close at 1732473940753 (+1 ms)Writing region close event to WAL at 1732473940753Closed at 1732473940753 2024-11-24T18:45:40,754 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T18:45:40,754 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T18:45:40,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T18:45:40,756 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T18:45:40,757 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T18:45:40,760 INFO [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(746): ClusterId : 0044d209-2696-4e94-8ad0-154f53912e1e 2024-11-24T18:45:40,760 INFO [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(746): ClusterId : 0044d209-2696-4e94-8ad0-154f53912e1e 2024-11-24T18:45:40,760 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T18:45:40,760 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T18:45:40,768 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T18:45:40,768 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T18:45:40,769 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T18:45:40,769 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T18:45:40,789 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T18:45:40,789 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T18:45:40,790 DEBUG [RS:0;f2b92657890a:40227 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e1828a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f2b92657890a/172.17.0.2:0 2024-11-24T18:45:40,791 DEBUG [RS:1;f2b92657890a:35197 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@97a65d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f2b92657890a/172.17.0.2:0 2024-11-24T18:45:40,802 DEBUG [RS:0;f2b92657890a:40227 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f2b92657890a:40227 2024-11-24T18:45:40,802 INFO [RS:0;f2b92657890a:40227 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T18:45:40,803 INFO [RS:0;f2b92657890a:40227 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T18:45:40,803 DEBUG [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T18:45:40,804 INFO [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(2659): reportForDuty to master=f2b92657890a,43953,1732473939979 with port=40227, startcode=1732473940146 2024-11-24T18:45:40,804 DEBUG [RS:0;f2b92657890a:40227 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T18:45:40,807 DEBUG [RS:1;f2b92657890a:35197 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f2b92657890a:35197 2024-11-24T18:45:40,807 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47869, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T18:45:40,807 INFO [RS:1;f2b92657890a:35197 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T18:45:40,807 INFO [RS:1;f2b92657890a:35197 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T18:45:40,807 DEBUG [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T18:45:40,808 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43953 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f2b92657890a,40227,1732473940146 2024-11-24T18:45:40,808 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43953 {}] master.ServerManager(517): Registering regionserver=f2b92657890a,40227,1732473940146 2024-11-24T18:45:40,808 INFO [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(2659): reportForDuty to master=f2b92657890a,43953,1732473939979 with port=35197, startcode=1732473940212 2024-11-24T18:45:40,809 DEBUG [RS:1;f2b92657890a:35197 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T18:45:40,810 DEBUG [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9 2024-11-24T18:45:40,810 DEBUG [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T18:45:40,810 DEBUG [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T18:45:40,812 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39175, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T18:45:40,812 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43953 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f2b92657890a,35197,1732473940212 2024-11-24T18:45:40,812 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43953 {}] master.ServerManager(517): Registering regionserver=f2b92657890a,35197,1732473940212 2024-11-24T18:45:40,814 DEBUG [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9 2024-11-24T18:45:40,814 DEBUG [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T18:45:40,814 DEBUG [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T18:45:40,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T18:45:40,858 DEBUG [RS:0;f2b92657890a:40227 {}] zookeeper.ZKUtil(111): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f2b92657890a,40227,1732473940146 2024-11-24T18:45:40,858 WARN [RS:0;f2b92657890a:40227 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T18:45:40,858 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f2b92657890a,40227,1732473940146] 2024-11-24T18:45:40,858 INFO [RS:0;f2b92657890a:40227 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T18:45:40,858 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f2b92657890a,35197,1732473940212] 2024-11-24T18:45:40,858 DEBUG [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,40227,1732473940146 2024-11-24T18:45:40,858 DEBUG [RS:1;f2b92657890a:35197 {}] zookeeper.ZKUtil(111): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f2b92657890a,35197,1732473940212 2024-11-24T18:45:40,858 WARN [RS:1;f2b92657890a:35197 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T18:45:40,858 INFO [RS:1;f2b92657890a:35197 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T18:45:40,859 DEBUG [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,35197,1732473940212 2024-11-24T18:45:40,869 INFO [RS:0;f2b92657890a:40227 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T18:45:40,872 INFO [RS:1;f2b92657890a:35197 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T18:45:40,873 INFO [RS:0;f2b92657890a:40227 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T18:45:40,873 INFO [RS:0;f2b92657890a:40227 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T18:45:40,873 INFO [RS:0;f2b92657890a:40227 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,873 INFO [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T18:45:40,874 INFO [RS:0;f2b92657890a:40227 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T18:45:40,874 INFO [RS:0;f2b92657890a:40227 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,875 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,875 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,875 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,875 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,875 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,875 INFO [RS:1;f2b92657890a:35197 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T18:45:40,875 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f2b92657890a:0, corePoolSize=2, maxPoolSize=2 2024-11-24T18:45:40,875 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,875 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,875 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,876 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,876 INFO [RS:1;f2b92657890a:35197 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T18:45:40,876 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,876 INFO [RS:1;f2b92657890a:35197 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,876 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,876 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f2b92657890a:0, corePoolSize=3, maxPoolSize=3 2024-11-24T18:45:40,876 DEBUG [RS:0;f2b92657890a:40227 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f2b92657890a:0, corePoolSize=3, maxPoolSize=3 2024-11-24T18:45:40,876 INFO [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T18:45:40,877 INFO [RS:1;f2b92657890a:35197 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T18:45:40,877 INFO [RS:1;f2b92657890a:35197 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,877 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,877 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,877 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,877 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,877 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,878 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f2b92657890a:0, corePoolSize=2, maxPoolSize=2 2024-11-24T18:45:40,878 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,878 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,878 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,878 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,878 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,878 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:40,878 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f2b92657890a:0, corePoolSize=3, maxPoolSize=3 2024-11-24T18:45:40,878 DEBUG [RS:1;f2b92657890a:35197 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f2b92657890a:0, corePoolSize=3, maxPoolSize=3 2024-11-24T18:45:40,879 INFO [RS:0;f2b92657890a:40227 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,879 INFO [RS:0;f2b92657890a:40227 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,879 INFO [RS:0;f2b92657890a:40227 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,879 INFO [RS:0;f2b92657890a:40227 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,879 INFO [RS:0;f2b92657890a:40227 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,879 INFO [RS:0;f2b92657890a:40227 {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,40227,1732473940146-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T18:45:40,885 INFO [RS:1;f2b92657890a:35197 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,885 INFO [RS:1;f2b92657890a:35197 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,885 INFO [RS:1;f2b92657890a:35197 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,885 INFO [RS:1;f2b92657890a:35197 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,885 INFO [RS:1;f2b92657890a:35197 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,885 INFO [RS:1;f2b92657890a:35197 {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,35197,1732473940212-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T18:45:40,902 INFO [RS:0;f2b92657890a:40227 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T18:45:40,902 INFO [RS:0;f2b92657890a:40227 {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,40227,1732473940146-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,902 INFO [RS:0;f2b92657890a:40227 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,902 INFO [RS:0;f2b92657890a:40227 {}] regionserver.Replication(171): f2b92657890a,40227,1732473940146 started 2024-11-24T18:45:40,905 INFO [RS:1;f2b92657890a:35197 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T18:45:40,905 INFO [RS:1;f2b92657890a:35197 {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,35197,1732473940212-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,906 INFO [RS:1;f2b92657890a:35197 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,906 INFO [RS:1;f2b92657890a:35197 {}] regionserver.Replication(171): f2b92657890a,35197,1732473940212 started 2024-11-24T18:45:40,907 WARN [f2b92657890a:43953 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T18:45:40,921 INFO [RS:1;f2b92657890a:35197 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,921 INFO [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(1482): Serving as f2b92657890a,35197,1732473940212, RpcServer on f2b92657890a/172.17.0.2:35197, sessionid=0x1016e2a63a20002 2024-11-24T18:45:40,921 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T18:45:40,921 DEBUG [RS:1;f2b92657890a:35197 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f2b92657890a,35197,1732473940212 2024-11-24T18:45:40,921 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f2b92657890a,35197,1732473940212' 2024-11-24T18:45:40,921 INFO [RS:0;f2b92657890a:40227 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:40,922 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T18:45:40,922 INFO [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(1482): Serving as f2b92657890a,40227,1732473940146, RpcServer on f2b92657890a/172.17.0.2:40227, sessionid=0x1016e2a63a20001 2024-11-24T18:45:40,922 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T18:45:40,922 DEBUG [RS:0;f2b92657890a:40227 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f2b92657890a,40227,1732473940146 2024-11-24T18:45:40,922 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f2b92657890a,40227,1732473940146' 2024-11-24T18:45:40,922 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T18:45:40,922 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T18:45:40,923 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T18:45:40,923 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T18:45:40,923 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T18:45:40,923 DEBUG [RS:1;f2b92657890a:35197 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f2b92657890a,35197,1732473940212 2024-11-24T18:45:40,923 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T18:45:40,923 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f2b92657890a,35197,1732473940212' 2024-11-24T18:45:40,923 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T18:45:40,923 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T18:45:40,923 DEBUG [RS:0;f2b92657890a:40227 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f2b92657890a,40227,1732473940146 2024-11-24T18:45:40,923 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f2b92657890a,40227,1732473940146' 2024-11-24T18:45:40,923 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T18:45:40,923 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T18:45:40,923 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T18:45:40,924 DEBUG [RS:1;f2b92657890a:35197 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T18:45:40,924 DEBUG [RS:0;f2b92657890a:40227 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T18:45:40,924 INFO [RS:0;f2b92657890a:40227 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T18:45:40,924 INFO [RS:1;f2b92657890a:35197 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T18:45:40,924 INFO [RS:0;f2b92657890a:40227 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T18:45:40,924 INFO [RS:1;f2b92657890a:35197 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T18:45:41,025 INFO [RS:0;f2b92657890a:40227 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T18:45:41,025 INFO [RS:1;f2b92657890a:35197 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T18:45:41,025 INFO [RS:0;f2b92657890a:40227 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f2b92657890a%2C40227%2C1732473940146, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,40227,1732473940146, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs, maxLogs=55 2024-11-24T18:45:41,026 INFO [RS:1;f2b92657890a:35197 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f2b92657890a%2C35197%2C1732473940212, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,35197,1732473940212, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs, maxLogs=55 2024-11-24T18:45:41,047 INFO [RS:1;f2b92657890a:35197 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,35197,1732473940212/f2b92657890a%2C35197%2C1732473940212.1732473941027 2024-11-24T18:45:41,047 INFO [RS:0;f2b92657890a:40227 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,40227,1732473940146/f2b92657890a%2C40227%2C1732473940146.1732473941027 2024-11-24T18:45:41,047 DEBUG [RS:1;f2b92657890a:35197 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T18:45:41,052 DEBUG [RS:0;f2b92657890a:40227 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T18:45:41,157 DEBUG [f2b92657890a:43953 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T18:45:41,157 DEBUG [f2b92657890a:43953 {}] balancer.BalancerClusterState(204): Hosts are {f2b92657890a=0} racks are {/default-rack=0} 2024-11-24T18:45:41,158 DEBUG [f2b92657890a:43953 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T18:45:41,158 DEBUG [f2b92657890a:43953 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T18:45:41,158 DEBUG [f2b92657890a:43953 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T18:45:41,158 DEBUG [f2b92657890a:43953 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T18:45:41,158 INFO [f2b92657890a:43953 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T18:45:41,158 INFO [f2b92657890a:43953 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T18:45:41,158 DEBUG [f2b92657890a:43953 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T18:45:41,159 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f2b92657890a,40227,1732473940146 2024-11-24T18:45:41,160 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f2b92657890a,40227,1732473940146, state=OPENING 2024-11-24T18:45:41,178 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T18:45:41,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:41,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:41,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:41,189 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T18:45:41,189 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T18:45:41,189 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T18:45:41,189 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T18:45:41,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f2b92657890a,40227,1732473940146}] 2024-11-24T18:45:41,342 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T18:45:41,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54809, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T18:45:41,349 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T18:45:41,349 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T18:45:41,350 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T18:45:41,352 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f2b92657890a%2C40227%2C1732473940146.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,40227,1732473940146, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs, maxLogs=55 2024-11-24T18:45:41,369 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,40227,1732473940146/f2b92657890a%2C40227%2C1732473940146.meta.1732473941352.meta 2024-11-24T18:45:41,369 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T18:45:41,370 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T18:45:41,370 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T18:45:41,370 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T18:45:41,370 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T18:45:41,370 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T18:45:41,370 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:41,370 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T18:45:41,370 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T18:45:41,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T18:45:41,372 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T18:45:41,372 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:41,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:41,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T18:45:41,373 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T18:45:41,373 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:41,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:41,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T18:45:41,373 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T18:45:41,373 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:41,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:41,374 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T18:45:41,374 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T18:45:41,374 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:41,374 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T18:45:41,374 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T18:45:41,375 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740 2024-11-24T18:45:41,375 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740 2024-11-24T18:45:41,376 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T18:45:41,376 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T18:45:41,376 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T18:45:41,377 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T18:45:41,377 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65850643, jitterRate=-0.018748953938484192}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T18:45:41,377 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T18:45:41,378 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732473941371Writing region info on filesystem at 1732473941371Initializing all the Stores at 1732473941371Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473941371Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473941371Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473941371Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732473941371Cleaning up temporary data from old regions at 1732473941376 (+5 ms)Running coprocessor post-open hooks at 1732473941377 (+1 ms)Region opened successfully at 1732473941378 (+1 ms) 2024-11-24T18:45:41,379 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732473941342 2024-11-24T18:45:41,381 DEBUG [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T18:45:41,382 INFO [RS_OPEN_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T18:45:41,382 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f2b92657890a,40227,1732473940146 2024-11-24T18:45:41,383 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f2b92657890a,40227,1732473940146, state=OPEN 2024-11-24T18:45:41,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T18:45:41,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T18:45:41,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T18:45:41,388 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f2b92657890a,40227,1732473940146 2024-11-24T18:45:41,388 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T18:45:41,388 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T18:45:41,388 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T18:45:41,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T18:45:41,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f2b92657890a,40227,1732473940146 in 199 msec 2024-11-24T18:45:41,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T18:45:41,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 636 msec 2024-11-24T18:45:41,394 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T18:45:41,394 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T18:45:41,398 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T18:45:41,398 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f2b92657890a,40227,1732473940146, seqNum=-1] 2024-11-24T18:45:41,399 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T18:45:41,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43283, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T18:45:41,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 731 msec 2024-11-24T18:45:41,408 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732473941408, completionTime=-1 2024-11-24T18:45:41,408 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T18:45:41,408 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T18:45:41,412 INFO [master/f2b92657890a:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T18:45:41,412 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732474001412 2024-11-24T18:45:41,412 INFO [master/f2b92657890a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732474061412 2024-11-24T18:45:41,412 INFO [master/f2b92657890a:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-24T18:45:41,412 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,43953,1732473939979-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:41,413 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,43953,1732473939979-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:41,413 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,43953,1732473939979-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:41,413 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f2b92657890a:43953, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:41,413 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:41,414 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:41,416 DEBUG [master/f2b92657890a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T18:45:41,419 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.114sec 2024-11-24T18:45:41,419 INFO [master/f2b92657890a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T18:45:41,419 INFO [master/f2b92657890a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T18:45:41,420 INFO [master/f2b92657890a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T18:45:41,420 INFO [master/f2b92657890a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T18:45:41,420 INFO [master/f2b92657890a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T18:45:41,420 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,43953,1732473939979-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T18:45:41,420 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,43953,1732473939979-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T18:45:41,421 DEBUG [master/f2b92657890a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T18:45:41,421 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T18:45:41,421 INFO [master/f2b92657890a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,43953,1732473939979-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:41,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T18:45:41,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f2b92657890a,43953,-1 for getting cluster id 2024-11-24T18:45:41,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T18:45:41,463 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0044d209-2696-4e94-8ad0-154f53912e1e' 2024-11-24T18:45:41,463 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T18:45:41,463 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0044d209-2696-4e94-8ad0-154f53912e1e" 2024-11-24T18:45:41,464 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@144884ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T18:45:41,464 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f2b92657890a,43953,-1] 2024-11-24T18:45:41,464 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T18:45:41,465 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:41,467 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T18:45:41,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af0bbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T18:45:41,469 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T18:45:41,470 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f2b92657890a,40227,1732473940146, seqNum=-1] 2024-11-24T18:45:41,471 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T18:45:41,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T18:45:41,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f2b92657890a,43953,1732473939979 2024-11-24T18:45:41,477 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T18:45:41,478 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f2b92657890a,43953,1732473939979 2024-11-24T18:45:41,478 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1183ecfc 2024-11-24T18:45:41,478 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T18:45:41,480 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T18:45:41,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43953 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732473941476', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T18:45:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43953 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732473941476 2024-11-24T18:45:41,483 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732473941476 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T18:45:41,484 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:41,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43953 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732473941476" procId is: 4 2024-11-24T18:45:41,484 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732473941476 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T18:45:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43953 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T18:45:41,494 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732473941476-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3f1629b951a5aa7d644e2d98072f2757, NAME => 'testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732473941476', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9 2024-11-24T18:45:41,500 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732473941476-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:41,500 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732473941476-pool-0 {}] regionserver.HRegion(1722): Closing 3f1629b951a5aa7d644e2d98072f2757, disabling compactions & flushes 2024-11-24T18:45:41,500 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732473941476-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:41,500 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732473941476-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:41,500 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732473941476-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. after waiting 0 ms 2024-11-24T18:45:41,500 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732473941476-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:41,500 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732473941476-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:41,500 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732473941476-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3f1629b951a5aa7d644e2d98072f2757: Waiting for close lock at 1732473941500Disabling compacts and flushes for region at 1732473941500Disabling writes for close at 1732473941500Writing region close event to WAL at 1732473941500Closed at 1732473941500 2024-11-24T18:45:41,501 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732473941476 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T18:45:41,502 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732473941501"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732473941501"}]},"ts":"1732473941501"} 2024-11-24T18:45:41,504 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T18:45:41,505 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732473941476 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T18:45:41,505 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732473941476","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732473941505"}]},"ts":"1732473941505"} 2024-11-24T18:45:41,507 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732473941476, state=ENABLING in hbase:meta 2024-11-24T18:45:41,508 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f2b92657890a=0} racks are {/default-rack=0} 2024-11-24T18:45:41,508 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T18:45:41,508 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T18:45:41,508 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T18:45:41,508 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T18:45:41,508 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T18:45:41,508 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T18:45:41,508 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T18:45:41,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732473941476, region=3f1629b951a5aa7d644e2d98072f2757, ASSIGN}] 2024-11-24T18:45:41,510 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732473941476, region=3f1629b951a5aa7d644e2d98072f2757, ASSIGN 2024-11-24T18:45:41,510 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732473941476, region=3f1629b951a5aa7d644e2d98072f2757, ASSIGN; state=OFFLINE, location=f2b92657890a,35197,1732473940212; forceNewPlan=false, retain=false 2024-11-24T18:45:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43953 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T18:45:41,661 INFO [f2b92657890a:43953 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T18:45:41,662 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3f1629b951a5aa7d644e2d98072f2757, regionState=OPENING, regionLocation=f2b92657890a,35197,1732473940212 2024-11-24T18:45:41,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732473941476, region=3f1629b951a5aa7d644e2d98072f2757, ASSIGN because future has completed 2024-11-24T18:45:41,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f1629b951a5aa7d644e2d98072f2757, server=f2b92657890a,35197,1732473940212}] 2024-11-24T18:45:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43953 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T18:45:41,823 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T18:45:41,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45905, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T18:45:41,831 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:41,832 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f1629b951a5aa7d644e2d98072f2757, NAME => 'testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757.', STARTKEY => '', ENDKEY => ''} 2024-11-24T18:45:41,833 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732473941476 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:41,833 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:41,833 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:41,833 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:41,835 INFO [StoreOpener-3f1629b951a5aa7d644e2d98072f2757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:41,841 INFO [StoreOpener-3f1629b951a5aa7d644e2d98072f2757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f1629b951a5aa7d644e2d98072f2757 columnFamilyName family 2024-11-24T18:45:41,841 DEBUG [StoreOpener-3f1629b951a5aa7d644e2d98072f2757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:41,841 INFO [StoreOpener-3f1629b951a5aa7d644e2d98072f2757-1 {}] regionserver.HStore(327): Store=3f1629b951a5aa7d644e2d98072f2757/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T18:45:41,841 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:41,842 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:41,842 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:41,842 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:41,842 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:41,843 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:41,848 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T18:45:41,849 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3f1629b951a5aa7d644e2d98072f2757; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70730925, jitterRate=0.053972914814949036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T18:45:41,849 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:41,850 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3f1629b951a5aa7d644e2d98072f2757: Running coprocessor pre-open hook at 1732473941834Writing region info on filesystem at 1732473941834Initializing all the Stores at 1732473941834Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473941834Cleaning up temporary data from old regions at 1732473941842 (+8 ms)Running coprocessor post-open hooks at 1732473941849 (+7 ms)Region opened successfully at 1732473941850 (+1 ms) 2024-11-24T18:45:41,850 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757., pid=6, masterSystemTime=1732473941822 2024-11-24T18:45:41,853 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:41,853 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:41,854 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3f1629b951a5aa7d644e2d98072f2757, regionState=OPEN, openSeqNum=2, regionLocation=f2b92657890a,35197,1732473940212 2024-11-24T18:45:41,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f1629b951a5aa7d644e2d98072f2757, server=f2b92657890a,35197,1732473940212 because future has completed 2024-11-24T18:45:41,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T18:45:41,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f1629b951a5aa7d644e2d98072f2757, server=f2b92657890a,35197,1732473940212 in 190 msec 2024-11-24T18:45:41,861 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T18:45:41,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T18:45:41,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732473941476, region=3f1629b951a5aa7d644e2d98072f2757, ASSIGN in 352 msec 2024-11-24T18:45:41,863 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732473941476 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T18:45:41,864 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732473941476","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732473941863"}]},"ts":"1732473941863"} 2024-11-24T18:45:41,866 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732473941476, state=ENABLED in hbase:meta 2024-11-24T18:45:41,867 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732473941476 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T18:45:41,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732473941476 in 386 msec 2024-11-24T18:45:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43953 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T18:45:42,117 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732473941476 completed 2024-11-24T18:45:42,117 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732473941476 get assigned. Timeout = 60000ms 2024-11-24T18:45:42,117 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T18:45:42,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732473941476 assigned to meta. Checking AM states. 2024-11-24T18:45:42,123 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T18:45:42,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732473941476 assigned. 2024-11-24T18:45:42,129 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732473941476', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757., hostname=f2b92657890a,35197,1732473940212, seqNum=2] 2024-11-24T18:45:42,129 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T18:45:42,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T18:45:42,135 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3f1629b951a5aa7d644e2d98072f2757 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T18:45:42,144 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f2aa7c4dcaec460e93e52f8cbe314942 2024-11-24T18:45:42,157 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/.tmp/family/f2aa7c4dcaec460e93e52f8cbe314942 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T18:45:42,158 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/.tmp/family/f2aa7c4dcaec460e93e52f8cbe314942 2024-11-24T18:45:42,160 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/.tmp/family/f2aa7c4dcaec460e93e52f8cbe314942 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942 2024-11-24T18:45:42,161 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942, delay=901 ms 2024-11-24T18:45:42,162 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T18:45:42,163 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 3f1629b951a5aa7d644e2d98072f2757 in 28ms, sequenceid=6, compaction requested=false 2024-11-24T18:45:42,163 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732473941476' 2024-11-24T18:45:42,164 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3f1629b951a5aa7d644e2d98072f2757: 2024-11-24T18:45:43,063 DEBUG [hfile-prefetch-1732473939182 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f2aa7c4dcaec460e93e52f8cbe314942_0. 2024-11-24T18:45:43,064 DEBUG [hfile-prefetch-1732473939182 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f2aa7c4dcaec460e93e52f8cbe314942_117. 2024-11-24T18:45:43,065 DEBUG [hfile-prefetch-1732473939182 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942, with total blocks 2, and data blocks 1 2024-11-24T18:45:43,065 DEBUG [hfile-prefetch-1732473939182 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T18:45:43,065 DEBUG [hfile-prefetch-1732473939182 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T18:45:43,065 DEBUG [hfile-prefetch-1732473939182 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942. Will sleep 100ms and try the verification again. 2024-11-24T18:45:43,164 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;f2b92657890a:35197,5,FailOnTimeoutGroup] 2024-11-24T18:45:43,164 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f2b92657890a,35197,1732473940212' ***** 2024-11-24T18:45:43,164 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-24T18:45:43,165 INFO [RS:1;f2b92657890a:35197 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T18:45:43,166 DEBUG [hfile-prefetch-1732473939182 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942, with total blocks 2, and data blocks 1 2024-11-24T18:45:43,166 DEBUG [hfile-prefetch-1732473939182 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T18:45:43,166 DEBUG [hfile-prefetch-1732473939182 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T18:45:43,166 DEBUG [hfile-prefetch-1732473939182 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942. Will sleep 100ms and try the verification again. 2024-11-24T18:45:43,170 INFO [RS:1;f2b92657890a:35197 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.persistence 2024-11-24T18:45:43,170 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T18:45:43,170 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T18:45:43,171 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T18:45:43,171 INFO [RS:1;f2b92657890a:35197 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T18:45:43,171 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T18:45:43,175 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T18:45:43,175 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T18:45:43,175 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T18:45:43,175 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T18:45:43,178 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T18:45:43,178 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732473943175 2024-11-24T18:45:43,178 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T18:45:43,181 DEBUG [RS:1;f2b92657890a:35197 {}] bucket.BucketCache(1487): Thread RS:1;f2b92657890a:35197 started persisting bucket cache to file 2024-11-24T18:45:43,181 DEBUG [RS:1;f2b92657890a:35197 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T18:45:43,181 DEBUG [RS:1;f2b92657890a:35197 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T18:45:43,184 DEBUG [RS:1;f2b92657890a:35197 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T18:45:43,184 DEBUG [RS:1;f2b92657890a:35197 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732473943181 2024-11-24T18:45:43,185 DEBUG [RS:1;f2b92657890a:35197 {}] bucket.BucketCache(1509): Thread RS:1;f2b92657890a:35197 finished persisting bucket cache to file, renaming 2024-11-24T18:45:43,185 INFO [RS:1;f2b92657890a:35197 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T18:45:43,185 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T18:45:43,185 INFO [RS:1;f2b92657890a:35197 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T18:45:43,185 INFO [RS:1;f2b92657890a:35197 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T18:45:43,185 INFO [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(3091): Received CLOSE for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,185 INFO [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(959): stopping server f2b92657890a,35197,1732473940212 2024-11-24T18:45:43,186 INFO [RS:1;f2b92657890a:35197 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T18:45:43,186 INFO [RS:1;f2b92657890a:35197 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f2b92657890a:35197. 2024-11-24T18:45:43,186 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T18:45:43,186 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3f1629b951a5aa7d644e2d98072f2757, disabling compactions & flushes 2024-11-24T18:45:43,186 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:43,186 DEBUG [RS:1;f2b92657890a:35197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T18:45:43,186 DEBUG [RS:1;f2b92657890a:35197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:43,186 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:43,186 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. after waiting 0 ms 2024-11-24T18:45:43,186 INFO [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T18:45:43,186 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:43,186 DEBUG [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(1325): Online Regions={3f1629b951a5aa7d644e2d98072f2757=testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757.} 2024-11-24T18:45:43,186 DEBUG [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(1351): Waiting on 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,187 DEBUG [StoreFileCloser-3f1629b951a5aa7d644e2d98072f2757-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942 2024-11-24T18:45:43,187 WARN [hfile-prefetch-1732473939182 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 10 more 2024-11-24T18:45:43,188 INFO [regionserver/f2b92657890a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T18:45:43,188 DEBUG [hfile-prefetch-1732473939182 {}] hfile.PrefetchExecutor(121): Prefetch completed for f2aa7c4dcaec460e93e52f8cbe314942 2024-11-24T18:45:43,191 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T18:45:43,191 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:43,191 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3f1629b951a5aa7d644e2d98072f2757: Waiting for close lock at 1732473943186Running coprocessor pre-close hooks at 1732473943186Disabling compacts and flushes for region at 1732473943186Disabling writes for close at 1732473943186Writing region close event to WAL at 1732473943187 (+1 ms)Running coprocessor post-close hooks at 1732473943191 (+4 ms)Closed at 1732473943191 2024-11-24T18:45:43,191 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:43,387 INFO [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(976): stopping server f2b92657890a,35197,1732473940212; all regions closed. 2024-11-24T18:45:43,389 DEBUG [RS:1;f2b92657890a:35197 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs 2024-11-24T18:45:43,389 INFO [RS:1;f2b92657890a:35197 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f2b92657890a%2C35197%2C1732473940212:(num 1732473941027) 2024-11-24T18:45:43,389 DEBUG [RS:1;f2b92657890a:35197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:43,389 INFO [RS:1;f2b92657890a:35197 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T18:45:43,390 INFO [RS:1;f2b92657890a:35197 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T18:45:43,390 INFO [RS:1;f2b92657890a:35197 {}] hbase.ChoreService(370): Chore service for: regionserver/f2b92657890a:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T18:45:43,390 INFO [RS:1;f2b92657890a:35197 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T18:45:43,390 INFO [RS:1;f2b92657890a:35197 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T18:45:43,390 INFO [regionserver/f2b92657890a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T18:45:43,390 INFO [RS:1;f2b92657890a:35197 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T18:45:43,390 INFO [RS:1;f2b92657890a:35197 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T18:45:43,391 INFO [RS:1;f2b92657890a:35197 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35197 2024-11-24T18:45:43,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f2b92657890a,35197,1732473940212 2024-11-24T18:45:43,409 INFO [RS:1;f2b92657890a:35197 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T18:45:43,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T18:45:43,420 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f2b92657890a,35197,1732473940212] 2024-11-24T18:45:43,430 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f2b92657890a,35197,1732473940212 already deleted, retry=false 2024-11-24T18:45:43,430 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of f2b92657890a,35197,1732473940212 on f2b92657890a,43953,1732473939979 2024-11-24T18:45:43,438 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f2b92657890a,35197,1732473940212, splitWal=true, meta=false 2024-11-24T18:45:43,441 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure f2b92657890a,35197,1732473940212, splitWal=true, meta=false 2024-11-24T18:45:43,441 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for f2b92657890a,35197,1732473940212 (carryingMeta=false) f2b92657890a,35197,1732473940212/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@563a609a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T18:45:43,442 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): f2b92657890a,35197,1732473940212 had 1 regions 2024-11-24T18:45:43,443 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f2b92657890a,35197,1732473940212, splitWal=true, meta=false, isMeta: false 2024-11-24T18:45:43,443 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,35197,1732473940212-splitting 2024-11-24T18:45:43,443 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,35197,1732473940212-splitting dir is empty, no logs to split. 2024-11-24T18:45:43,444 INFO [PEWorker-1 {}] master.SplitWALManager(105): f2b92657890a,35197,1732473940212 WAL count=0, meta=false 2024-11-24T18:45:43,445 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,35197,1732473940212-splitting dir is empty, no logs to split. 2024-11-24T18:45:43,445 INFO [PEWorker-1 {}] master.SplitWALManager(105): f2b92657890a,35197,1732473940212 WAL count=0, meta=false 2024-11-24T18:45:43,445 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if f2b92657890a,35197,1732473940212 WAL splitting is done? wals=0, meta=false 2024-11-24T18:45:43,446 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for f2b92657890a,35197,1732473940212 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,35197,1732473940212-splitting does not exist 2024-11-24T18:45:43,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732473941476, region=3f1629b951a5aa7d644e2d98072f2757, ASSIGN}] 2024-11-24T18:45:43,448 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732473941476, region=3f1629b951a5aa7d644e2d98072f2757, ASSIGN 2024-11-24T18:45:43,449 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732473941476, region=3f1629b951a5aa7d644e2d98072f2757, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T18:45:43,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:43,520 INFO [RS:1;f2b92657890a:35197 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T18:45:43,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35197-0x1016e2a63a20002, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:43,520 INFO [RS:1;f2b92657890a:35197 {}] regionserver.HRegionServer(1031): Exiting; stopping=f2b92657890a,35197,1732473940212; zookeeper connection closed. 2024-11-24T18:45:43,521 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@38a98be3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@38a98be3 2024-11-24T18:45:43,600 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=3f1629b951a5aa7d644e2d98072f2757, regionState=OPENING, regionLocation=f2b92657890a,40227,1732473940146 2024-11-24T18:45:43,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732473941476, region=3f1629b951a5aa7d644e2d98072f2757, ASSIGN because future has completed 2024-11-24T18:45:43,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f1629b951a5aa7d644e2d98072f2757, server=f2b92657890a,40227,1732473940146}] 2024-11-24T18:45:43,678 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f2b92657890a:0 server-side Connection retries=45 2024-11-24T18:45:43,679 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:43,679 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:43,679 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T18:45:43,679 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T18:45:43,679 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T18:45:43,679 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T18:45:43,679 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T18:45:43,680 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42299 2024-11-24T18:45:43,682 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42299 connecting to ZooKeeper ensemble=127.0.0.1:60450 2024-11-24T18:45:43,682 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:43,682 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:43,693 DEBUG [pool-297-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422990x0, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T18:45:43,694 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42299-0x1016e2a63a20003 connected 2024-11-24T18:45:43,694 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-24T18:45:43,694 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:42299-0x1016e2a63a20003, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-24T18:45:43,695 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T18:45:43,696 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.cache 2024-11-24T18:45:43,696 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T18:45:43,696 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T18:45:43,696 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T18:45:43,696 INFO [Thread-199 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T18:45:43,697 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T18:45:43,697 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T18:45:43,698 INFO [Thread-199 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-24T18:45:43,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:42299-0x1016e2a63a20003, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T18:45:43,698 INFO [Thread-199 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-24T18:45:43,700 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42299-0x1016e2a63a20003, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T18:45:43,700 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42299 2024-11-24T18:45:43,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42299 2024-11-24T18:45:43,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42299 2024-11-24T18:45:43,701 DEBUG [Thread-199 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-24T18:45:43,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42299 2024-11-24T18:45:43,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42299 2024-11-24T18:45:43,702 INFO [Thread-199 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T18:45:43,702 INFO [Thread-199 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-24T18:45:43,702 INFO [Thread-199 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.persistence is complete. 2024-11-24T18:45:43,704 INFO [RS:2;f2b92657890a:42299 {}] regionserver.HRegionServer(746): ClusterId : 0044d209-2696-4e94-8ad0-154f53912e1e 2024-11-24T18:45:43,704 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T18:45:43,715 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T18:45:43,715 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T18:45:43,723 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T18:45:43,723 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T18:45:43,726 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T18:45:43,726 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T18:45:43,726 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T18:45:43,726 DEBUG [RS:2;f2b92657890a:42299 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b58c5cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f2b92657890a/172.17.0.2:0 2024-11-24T18:45:43,727 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T18:45:43,727 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T18:45:43,736 DEBUG [RS:2;f2b92657890a:42299 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;f2b92657890a:42299 2024-11-24T18:45:43,736 INFO [RS:2;f2b92657890a:42299 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T18:45:43,736 INFO [RS:2;f2b92657890a:42299 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T18:45:43,736 DEBUG [RS:2;f2b92657890a:42299 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T18:45:43,737 INFO [RS:2;f2b92657890a:42299 {}] regionserver.HRegionServer(2659): reportForDuty to master=f2b92657890a,43953,1732473939979 with port=42299, startcode=1732473943678 2024-11-24T18:45:43,737 DEBUG [RS:2;f2b92657890a:42299 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T18:45:43,739 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41141, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T18:45:43,740 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43953 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f2b92657890a,42299,1732473943678 2024-11-24T18:45:43,740 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43953 {}] master.ServerManager(517): Registering regionserver=f2b92657890a,42299,1732473943678 2024-11-24T18:45:43,741 DEBUG [RS:2;f2b92657890a:42299 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9 2024-11-24T18:45:43,741 DEBUG [RS:2;f2b92657890a:42299 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T18:45:43,741 DEBUG [RS:2;f2b92657890a:42299 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T18:45:43,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T18:45:43,746 DEBUG [RS:2;f2b92657890a:42299 {}] zookeeper.ZKUtil(111): regionserver:42299-0x1016e2a63a20003, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f2b92657890a,42299,1732473943678 2024-11-24T18:45:43,746 WARN [RS:2;f2b92657890a:42299 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T18:45:43,746 INFO [RS:2;f2b92657890a:42299 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T18:45:43,746 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f2b92657890a,42299,1732473943678] 2024-11-24T18:45:43,747 DEBUG [RS:2;f2b92657890a:42299 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,42299,1732473943678 2024-11-24T18:45:43,751 INFO [RS:2;f2b92657890a:42299 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T18:45:43,754 INFO [RS:2;f2b92657890a:42299 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T18:45:43,755 INFO [RS:2;f2b92657890a:42299 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T18:45:43,755 INFO [RS:2;f2b92657890a:42299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:43,755 INFO [RS:2;f2b92657890a:42299 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T18:45:43,755 INFO [RS:2;f2b92657890a:42299 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T18:45:43,756 INFO [RS:2;f2b92657890a:42299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:43,756 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:43,756 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:43,756 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:43,756 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:43,756 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:43,756 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f2b92657890a:0, corePoolSize=2, maxPoolSize=2 2024-11-24T18:45:43,756 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:43,756 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:43,756 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:43,757 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:43,757 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:43,757 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f2b92657890a:0, corePoolSize=1, maxPoolSize=1 2024-11-24T18:45:43,757 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f2b92657890a:0, corePoolSize=3, maxPoolSize=3 2024-11-24T18:45:43,757 DEBUG [RS:2;f2b92657890a:42299 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f2b92657890a:0, corePoolSize=3, maxPoolSize=3 2024-11-24T18:45:43,757 INFO [RS:2;f2b92657890a:42299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:43,757 INFO [RS:2;f2b92657890a:42299 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:43,757 INFO [RS:2;f2b92657890a:42299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:43,757 INFO [RS:2;f2b92657890a:42299 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:43,757 INFO [RS:2;f2b92657890a:42299 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:43,758 INFO [RS:2;f2b92657890a:42299 {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,42299,1732473943678-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T18:45:43,764 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:43,765 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f1629b951a5aa7d644e2d98072f2757, NAME => 'testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757.', STARTKEY => '', ENDKEY => ''} 2024-11-24T18:45:43,765 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732473941476 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,765 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T18:45:43,765 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,765 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,766 INFO [StoreOpener-3f1629b951a5aa7d644e2d98072f2757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,767 INFO [StoreOpener-3f1629b951a5aa7d644e2d98072f2757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f1629b951a5aa7d644e2d98072f2757 columnFamilyName family 2024-11-24T18:45:43,767 DEBUG [StoreOpener-3f1629b951a5aa7d644e2d98072f2757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T18:45:43,768 DEBUG [StoreFileOpener-3f1629b951a5aa7d644e2d98072f2757-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942, delay=969 ms 2024-11-24T18:45:43,769 DEBUG [StoreOpener-3f1629b951a5aa7d644e2d98072f2757-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942 2024-11-24T18:45:43,769 INFO [StoreOpener-3f1629b951a5aa7d644e2d98072f2757-1 {}] regionserver.HStore(327): Store=3f1629b951a5aa7d644e2d98072f2757/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T18:45:43,769 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,770 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,771 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,771 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,771 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,772 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,773 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 3f1629b951a5aa7d644e2d98072f2757; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66295978, jitterRate=-0.012112945318222046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T18:45:43,773 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:43,774 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 3f1629b951a5aa7d644e2d98072f2757: Running coprocessor pre-open hook at 1732473943765Writing region info on filesystem at 1732473943765Initializing all the Stores at 1732473943766 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732473943766Cleaning up temporary data from old regions at 1732473943771 (+5 ms)Running coprocessor post-open hooks at 1732473943773 (+2 ms)Region opened successfully at 1732473943774 (+1 ms) 2024-11-24T18:45:43,775 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757., pid=9, masterSystemTime=1732473943762 2024-11-24T18:45:43,776 INFO [RS:2;f2b92657890a:42299 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T18:45:43,777 INFO [RS:2;f2b92657890a:42299 {}] hbase.ChoreService(168): Chore ScheduledChore name=f2b92657890a,42299,1732473943678-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:43,777 INFO [RS:2;f2b92657890a:42299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:43,777 DEBUG [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:43,777 INFO [RS:2;f2b92657890a:42299 {}] regionserver.Replication(171): f2b92657890a,42299,1732473943678 started 2024-11-24T18:45:43,777 INFO [RS_OPEN_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:43,778 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=3f1629b951a5aa7d644e2d98072f2757, regionState=OPEN, openSeqNum=10, regionLocation=f2b92657890a,40227,1732473940146 2024-11-24T18:45:43,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f1629b951a5aa7d644e2d98072f2757, server=f2b92657890a,40227,1732473940146 because future has completed 2024-11-24T18:45:43,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-24T18:45:43,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f1629b951a5aa7d644e2d98072f2757, server=f2b92657890a,40227,1732473940146 in 173 msec 2024-11-24T18:45:43,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T18:45:43,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732473941476, region=3f1629b951a5aa7d644e2d98072f2757, ASSIGN in 336 msec 2024-11-24T18:45:43,785 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server f2b92657890a,35197,1732473940212 after splitting done 2024-11-24T18:45:43,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure f2b92657890a,35197,1732473940212, splitWal=true, meta=false in 353 msec 2024-11-24T18:45:43,798 INFO [RS:2;f2b92657890a:42299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T18:45:43,798 INFO [RS:2;f2b92657890a:42299 {}] regionserver.HRegionServer(1482): Serving as f2b92657890a,42299,1732473943678, RpcServer on f2b92657890a/172.17.0.2:42299, sessionid=0x1016e2a63a20003 2024-11-24T18:45:43,798 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T18:45:43,798 DEBUG [RS:2;f2b92657890a:42299 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f2b92657890a,42299,1732473943678 2024-11-24T18:45:43,798 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f2b92657890a,42299,1732473943678' 2024-11-24T18:45:43,798 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T18:45:43,799 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T18:45:43,800 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T18:45:43,800 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T18:45:43,800 DEBUG [RS:2;f2b92657890a:42299 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f2b92657890a,42299,1732473943678 2024-11-24T18:45:43,800 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f2b92657890a,42299,1732473943678' 2024-11-24T18:45:43,800 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T18:45:43,801 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T18:45:43,801 DEBUG [RS:2;f2b92657890a:42299 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T18:45:43,801 INFO [RS:2;f2b92657890a:42299 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T18:45:43,801 INFO [RS:2;f2b92657890a:42299 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T18:45:43,902 INFO [RS:2;f2b92657890a:42299 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T18:45:43,904 INFO [RS:2;f2b92657890a:42299 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f2b92657890a%2C42299%2C1732473943678, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,42299,1732473943678, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs, maxLogs=55 2024-11-24T18:45:43,927 INFO [RS:2;f2b92657890a:42299 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/WALs/f2b92657890a,42299,1732473943678/f2b92657890a%2C42299%2C1732473943678.1732473943906 2024-11-24T18:45:43,928 DEBUG [RS:2;f2b92657890a:42299 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T18:45:44,299 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T18:45:44,299 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T18:45:44,299 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T18:45:44,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:44,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:44,299 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T18:45:44,299 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T18:45:44,300 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=771001981, stopped=false 2024-11-24T18:45:44,300 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f2b92657890a,43953,1732473939979 2024-11-24T18:45:44,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T18:45:44,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T18:45:44,314 DEBUG [pool-297-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42299-0x1016e2a63a20003, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T18:45:44,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:44,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:44,315 DEBUG [pool-297-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42299-0x1016e2a63a20003, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:44,315 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T18:45:44,315 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T18:45:44,316 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T18:45:44,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:44,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T18:45:44,316 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f2b92657890a,40227,1732473940146' ***** 2024-11-24T18:45:44,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T18:45:44,317 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T18:45:44,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42299-0x1016e2a63a20003, quorum=127.0.0.1:60450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T18:45:44,317 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f2b92657890a,42299,1732473943678' ***** 2024-11-24T18:45:44,317 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T18:45:44,317 INFO [RS:0;f2b92657890a:40227 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T18:45:44,317 INFO [RS:0;f2b92657890a:40227 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.persistence 2024-11-24T18:45:44,317 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T18:45:44,317 INFO [RS:2;f2b92657890a:42299 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T18:45:44,318 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T18:45:44,318 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T18:45:44,318 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T18:45:44,318 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T18:45:44,318 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T18:45:44,318 INFO [RS:2;f2b92657890a:42299 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f97f59c1-8f20-e6d0-187c-5e2bb63b3849/bucket.persistence 2024-11-24T18:45:44,319 INFO [RS:0;f2b92657890a:40227 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T18:45:44,319 INFO [RS:2;f2b92657890a:42299 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T18:45:44,319 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T18:45:44,319 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T18:45:44,329 DEBUG [RS:0;f2b92657890a:40227 {}] bucket.BucketCache(1487): Thread RS:0;f2b92657890a:40227 started persisting bucket cache to file 2024-11-24T18:45:44,329 DEBUG [RS:0;f2b92657890a:40227 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T18:45:44,329 DEBUG [RS:0;f2b92657890a:40227 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T18:45:44,329 DEBUG [RS:2;f2b92657890a:42299 {}] bucket.BucketCache(1487): Thread RS:2;f2b92657890a:42299 started persisting bucket cache to file 2024-11-24T18:45:44,329 DEBUG [RS:2;f2b92657890a:42299 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T18:45:44,329 DEBUG [RS:2;f2b92657890a:42299 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T18:45:44,332 DEBUG [RS:0;f2b92657890a:40227 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T18:45:44,332 DEBUG [RS:2;f2b92657890a:42299 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T18:45:44,332 DEBUG [RS:2;f2b92657890a:42299 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732473944329 2024-11-24T18:45:44,332 DEBUG [RS:0;f2b92657890a:40227 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732473944329 2024-11-24T18:45:44,332 DEBUG [RS:0;f2b92657890a:40227 {}] bucket.BucketCache(1509): Thread RS:0;f2b92657890a:40227 finished persisting bucket cache to file, renaming 2024-11-24T18:45:44,332 INFO [RS:0;f2b92657890a:40227 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T18:45:44,332 INFO [RS:0;f2b92657890a:40227 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T18:45:44,332 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T18:45:44,332 INFO [RS:0;f2b92657890a:40227 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T18:45:44,332 INFO [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(3091): Received CLOSE for 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:44,333 INFO [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(959): stopping server f2b92657890a,40227,1732473940146 2024-11-24T18:45:44,333 INFO [RS:0;f2b92657890a:40227 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T18:45:44,333 INFO [RS:0;f2b92657890a:40227 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f2b92657890a:40227. 2024-11-24T18:45:44,333 DEBUG [RS:0;f2b92657890a:40227 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T18:45:44,333 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T18:45:44,333 DEBUG [RS:0;f2b92657890a:40227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:44,333 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3f1629b951a5aa7d644e2d98072f2757, disabling compactions & flushes 2024-11-24T18:45:44,333 INFO [RS:0;f2b92657890a:40227 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T18:45:44,333 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:44,333 INFO [RS:0;f2b92657890a:40227 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T18:45:44,333 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:44,333 INFO [RS:0;f2b92657890a:40227 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T18:45:44,333 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. after waiting 0 ms 2024-11-24T18:45:44,333 INFO [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T18:45:44,333 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:44,333 INFO [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T18:45:44,333 DEBUG [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(1325): Online Regions={3f1629b951a5aa7d644e2d98072f2757=testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757., 1588230740=hbase:meta,,1.1588230740} 2024-11-24T18:45:44,333 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T18:45:44,333 DEBUG [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3f1629b951a5aa7d644e2d98072f2757 2024-11-24T18:45:44,333 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T18:45:44,334 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T18:45:44,334 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T18:45:44,334 DEBUG [StoreFileCloser-3f1629b951a5aa7d644e2d98072f2757-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/family/f2aa7c4dcaec460e93e52f8cbe314942 2024-11-24T18:45:44,334 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T18:45:44,334 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T18:45:44,334 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-24T18:45:44,337 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/default/testBlockEvictionOnGracefulStop1732473941476/3f1629b951a5aa7d644e2d98072f2757/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T18:45:44,338 INFO [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:44,338 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3f1629b951a5aa7d644e2d98072f2757: Waiting for close lock at 1732473944333Running coprocessor pre-close hooks at 1732473944333Disabling compacts and flushes for region at 1732473944333Disabling writes for close at 1732473944333Writing region close event to WAL at 1732473944334 (+1 ms)Running coprocessor post-close hooks at 1732473944338 (+4 ms)Closed at 1732473944338 2024-11-24T18:45:44,338 DEBUG [RS_CLOSE_REGION-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757. 2024-11-24T18:45:44,342 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3eefb661718b40dd988368edfc3949ab 2024-11-24T18:45:44,348 DEBUG [RS:2;f2b92657890a:42299 {}] bucket.BucketCache(1509): Thread RS:2;f2b92657890a:42299 finished persisting bucket cache to file, renaming 2024-11-24T18:45:44,348 WARN [RS:2;f2b92657890a:42299 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-24T18:45:44,348 INFO [RS:2;f2b92657890a:42299 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T18:45:44,348 INFO [RS:2;f2b92657890a:42299 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T18:45:44,348 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T18:45:44,348 INFO [RS:2;f2b92657890a:42299 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T18:45:44,348 INFO [RS:2;f2b92657890a:42299 {}] regionserver.HRegionServer(959): stopping server f2b92657890a,42299,1732473943678 2024-11-24T18:45:44,348 INFO [RS:2;f2b92657890a:42299 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T18:45:44,348 INFO [RS:2;f2b92657890a:42299 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;f2b92657890a:42299. 2024-11-24T18:45:44,348 DEBUG [RS:2;f2b92657890a:42299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T18:45:44,348 DEBUG [RS:2;f2b92657890a:42299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:44,349 INFO [RS:2;f2b92657890a:42299 {}] regionserver.HRegionServer(976): stopping server f2b92657890a,42299,1732473943678; all regions closed. 2024-11-24T18:45:44,350 DEBUG [RS:2;f2b92657890a:42299 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs 2024-11-24T18:45:44,350 INFO [RS:2;f2b92657890a:42299 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f2b92657890a%2C42299%2C1732473943678:(num 1732473943906) 2024-11-24T18:45:44,350 DEBUG [RS:2;f2b92657890a:42299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:44,350 INFO [RS:2;f2b92657890a:42299 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T18:45:44,351 INFO [RS:2;f2b92657890a:42299 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T18:45:44,351 INFO [RS:2;f2b92657890a:42299 {}] hbase.ChoreService(370): Chore service for: regionserver/f2b92657890a:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T18:45:44,351 INFO [RS:2;f2b92657890a:42299 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T18:45:44,351 INFO [regionserver/f2b92657890a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T18:45:44,351 INFO [RS:2;f2b92657890a:42299 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T18:45:44,351 INFO [RS:2;f2b92657890a:42299 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T18:45:44,351 INFO [RS:2;f2b92657890a:42299 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T18:45:44,351 INFO [RS:2;f2b92657890a:42299 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42299 2024-11-24T18:45:44,356 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/info/3eefb661718b40dd988368edfc3949ab is 209, key is testBlockEvictionOnGracefulStop1732473941476,,1732473941481.3f1629b951a5aa7d644e2d98072f2757./info:regioninfo/1732473943777/Put/seqid=0 2024-11-24T18:45:44,357 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/info/3eefb661718b40dd988368edfc3949ab 2024-11-24T18:45:44,358 INFO [regionserver/f2b92657890a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T18:45:44,364 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bc7cf1690d52461e9cfa02cd08181329 2024-11-24T18:45:44,375 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/ns/bc7cf1690d52461e9cfa02cd08181329 is 43, key is default/ns:d/1732473941402/Put/seqid=0 2024-11-24T18:45:44,376 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/ns/bc7cf1690d52461e9cfa02cd08181329 2024-11-24T18:45:44,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T18:45:44,377 DEBUG [pool-297-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42299-0x1016e2a63a20003, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f2b92657890a,42299,1732473943678 2024-11-24T18:45:44,377 INFO [RS:2;f2b92657890a:42299 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T18:45:44,378 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f2b92657890a,42299,1732473943678] 2024-11-24T18:45:44,383 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 04f42620e90b497e9dbd62dd06989376 2024-11-24T18:45:44,384 INFO [regionserver/f2b92657890a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T18:45:44,395 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/table/04f42620e90b497e9dbd62dd06989376 is 80, key is testBlockEvictionOnGracefulStop1732473941476/table:state/1732473941863/Put/seqid=0 2024-11-24T18:45:44,395 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/table/04f42620e90b497e9dbd62dd06989376 2024-11-24T18:45:44,397 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/info/3eefb661718b40dd988368edfc3949ab as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/info/3eefb661718b40dd988368edfc3949ab 2024-11-24T18:45:44,398 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f2b92657890a,42299,1732473943678 already deleted, retry=false 2024-11-24T18:45:44,398 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/info/3eefb661718b40dd988368edfc3949ab, delay=943 ms 2024-11-24T18:45:44,398 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f2b92657890a,42299,1732473943678 expired; onlineServers=1 2024-11-24T18:45:44,399 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/info/3eefb661718b40dd988368edfc3949ab, entries=14, sequenceid=13, filesize=7.5 K 2024-11-24T18:45:44,399 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/ns/bc7cf1690d52461e9cfa02cd08181329 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/ns/bc7cf1690d52461e9cfa02cd08181329 2024-11-24T18:45:44,400 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/ns/bc7cf1690d52461e9cfa02cd08181329, delay=938 ms 2024-11-24T18:45:44,401 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/ns/bc7cf1690d52461e9cfa02cd08181329, entries=2, sequenceid=13, filesize=5.0 K 2024-11-24T18:45:44,401 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/.tmp/table/04f42620e90b497e9dbd62dd06989376 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/table/04f42620e90b497e9dbd62dd06989376 2024-11-24T18:45:44,402 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/table/04f42620e90b497e9dbd62dd06989376, delay=984 ms 2024-11-24T18:45:44,403 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/table/04f42620e90b497e9dbd62dd06989376, entries=2, sequenceid=13, filesize=5.3 K 2024-11-24T18:45:44,404 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=13, compaction requested=false 2024-11-24T18:45:44,404 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/info/3eefb661718b40dd988368edfc3949ab 2024-11-24T18:45:44,405 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/ns/bc7cf1690d52461e9cfa02cd08181329 2024-11-24T18:45:44,407 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/table/04f42620e90b497e9dbd62dd06989376 2024-11-24T18:45:44,410 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-24T18:45:44,410 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T18:45:44,411 INFO [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T18:45:44,411 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732473944333Running coprocessor pre-close hooks at 1732473944333Disabling compacts and flushes for region at 1732473944333Disabling writes for close at 1732473944334 (+1 ms)Obtaining lock to block concurrent updates at 1732473944334Preparing flush snapshotting stores in 1588230740 at 1732473944334Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732473944334Flushing stores of hbase:meta,,1.1588230740 at 1732473944334Flushing 1588230740/info: creating writer at 1732473944335 (+1 ms)Flushing 1588230740/info: appending metadata at 1732473944356 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732473944356Flushing 1588230740/ns: creating writer at 1732473944359 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732473944375 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732473944375Flushing 1588230740/table: creating writer at 1732473944378 (+3 ms)Flushing 1588230740/table: appending metadata at 1732473944394 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732473944394Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45e0f6cf: reopening flushed file at 1732473944397 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bcc9e0e: reopening flushed file at 1732473944399 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b9d883f: reopening flushed file at 1732473944401 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=13, compaction requested=false at 1732473944404 (+3 ms)Writing region close event to WAL at 1732473944407 (+3 ms)Running coprocessor post-close hooks at 1732473944410 (+3 ms)Closed at 1732473944411 (+1 ms) 2024-11-24T18:45:44,411 DEBUG [RS_CLOSE_META-regionserver/f2b92657890a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T18:45:44,488 DEBUG [pool-297-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42299-0x1016e2a63a20003, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:44,488 INFO [RS:2;f2b92657890a:42299 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T18:45:44,488 DEBUG [pool-297-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42299-0x1016e2a63a20003, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:44,488 INFO [RS:2;f2b92657890a:42299 {}] regionserver.HRegionServer(1031): Exiting; stopping=f2b92657890a,42299,1732473943678; zookeeper connection closed. 2024-11-24T18:45:44,489 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@77b8a70b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@77b8a70b 2024-11-24T18:45:44,534 INFO [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(976): stopping server f2b92657890a,40227,1732473940146; all regions closed. 2024-11-24T18:45:44,539 DEBUG [RS:0;f2b92657890a:40227 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs 2024-11-24T18:45:44,539 INFO [RS:0;f2b92657890a:40227 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f2b92657890a%2C40227%2C1732473940146.meta:.meta(num 1732473941352) 2024-11-24T18:45:44,542 DEBUG [RS:0;f2b92657890a:40227 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/oldWALs 2024-11-24T18:45:44,542 INFO [RS:0;f2b92657890a:40227 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f2b92657890a%2C40227%2C1732473940146:(num 1732473941027) 2024-11-24T18:45:44,543 DEBUG [RS:0;f2b92657890a:40227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T18:45:44,543 INFO [RS:0;f2b92657890a:40227 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T18:45:44,543 INFO [RS:0;f2b92657890a:40227 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T18:45:44,543 INFO [RS:0;f2b92657890a:40227 {}] hbase.ChoreService(370): Chore service for: regionserver/f2b92657890a:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T18:45:44,544 INFO [RS:0;f2b92657890a:40227 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T18:45:44,544 INFO [regionserver/f2b92657890a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T18:45:44,544 INFO [RS:0;f2b92657890a:40227 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40227 2024-11-24T18:45:44,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T18:45:44,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f2b92657890a,40227,1732473940146 2024-11-24T18:45:44,556 INFO [RS:0;f2b92657890a:40227 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T18:45:44,557 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f2b92657890a,40227,1732473940146] 2024-11-24T18:45:44,577 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f2b92657890a,40227,1732473940146 already deleted, retry=false 2024-11-24T18:45:44,577 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f2b92657890a,40227,1732473940146 expired; onlineServers=0 2024-11-24T18:45:44,578 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f2b92657890a,43953,1732473939979' ***** 2024-11-24T18:45:44,578 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T18:45:44,578 INFO [M:0;f2b92657890a:43953 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T18:45:44,578 INFO [M:0;f2b92657890a:43953 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T18:45:44,578 DEBUG [M:0;f2b92657890a:43953 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T18:45:44,578 DEBUG [M:0;f2b92657890a:43953 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T18:45:44,578 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T18:45:44,578 DEBUG [master/f2b92657890a:0:becomeActiveMaster-HFileCleaner.small.0-1732473940698 {}] cleaner.HFileCleaner(306): Exit Thread[master/f2b92657890a:0:becomeActiveMaster-HFileCleaner.small.0-1732473940698,5,FailOnTimeoutGroup] 2024-11-24T18:45:44,578 DEBUG [master/f2b92657890a:0:becomeActiveMaster-HFileCleaner.large.0-1732473940697 {}] cleaner.HFileCleaner(306): Exit Thread[master/f2b92657890a:0:becomeActiveMaster-HFileCleaner.large.0-1732473940697,5,FailOnTimeoutGroup] 2024-11-24T18:45:44,579 INFO [M:0;f2b92657890a:43953 {}] hbase.ChoreService(370): Chore service for: master/f2b92657890a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T18:45:44,579 INFO [M:0;f2b92657890a:43953 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T18:45:44,579 DEBUG [M:0;f2b92657890a:43953 {}] master.HMaster(1795): Stopping service threads 2024-11-24T18:45:44,579 INFO [M:0;f2b92657890a:43953 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T18:45:44,579 INFO [M:0;f2b92657890a:43953 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T18:45:44,580 INFO [M:0;f2b92657890a:43953 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T18:45:44,580 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T18:45:44,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T18:45:44,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T18:45:44,588 DEBUG [M:0;f2b92657890a:43953 {}] zookeeper.ZKUtil(347): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T18:45:44,588 WARN [M:0;f2b92657890a:43953 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T18:45:44,589 INFO [M:0;f2b92657890a:43953 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/.lastflushedseqids 2024-11-24T18:45:44,598 INFO [M:0;f2b92657890a:43953 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T18:45:44,598 INFO [M:0;f2b92657890a:43953 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T18:45:44,598 DEBUG [M:0;f2b92657890a:43953 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T18:45:44,598 INFO [M:0;f2b92657890a:43953 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:44,598 DEBUG [M:0;f2b92657890a:43953 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:44,598 DEBUG [M:0;f2b92657890a:43953 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T18:45:44,598 DEBUG [M:0;f2b92657890a:43953 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:44,599 INFO [M:0;f2b92657890a:43953 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-11-24T18:45:44,607 DEBUG [M:0;f2b92657890a:43953 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9acbe6411bd749c7964e156cd5038fa3 2024-11-24T18:45:44,620 DEBUG [M:0;f2b92657890a:43953 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9acbe6411bd749c7964e156cd5038fa3 is 82, key is hbase:meta,,1/info:regioninfo/1732473941382/Put/seqid=0 2024-11-24T18:45:44,621 INFO [M:0;f2b92657890a:43953 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9acbe6411bd749c7964e156cd5038fa3 2024-11-24T18:45:44,627 DEBUG [M:0;f2b92657890a:43953 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2522b032cbe64ca8948f349a577570df 2024-11-24T18:45:44,638 DEBUG [M:0;f2b92657890a:43953 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2522b032cbe64ca8948f349a577570df is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732473941869/Put/seqid=0 2024-11-24T18:45:44,639 INFO [M:0;f2b92657890a:43953 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2522b032cbe64ca8948f349a577570df 2024-11-24T18:45:44,646 DEBUG [M:0;f2b92657890a:43953 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 47e18a20d91949559a78b7b9bc6d129c 2024-11-24T18:45:44,658 DEBUG [M:0;f2b92657890a:43953 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/47e18a20d91949559a78b7b9bc6d129c is 69, key is f2b92657890a,40227,1732473940146/rs:state/1732473940808/Put/seqid=0 2024-11-24T18:45:44,658 INFO [M:0;f2b92657890a:43953 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/47e18a20d91949559a78b7b9bc6d129c 2024-11-24T18:45:44,660 INFO [M:0;f2b92657890a:43953 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 47e18a20d91949559a78b7b9bc6d129c 2024-11-24T18:45:44,660 DEBUG [M:0;f2b92657890a:43953 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9acbe6411bd749c7964e156cd5038fa3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9acbe6411bd749c7964e156cd5038fa3 2024-11-24T18:45:44,662 INFO [M:0;f2b92657890a:43953 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9acbe6411bd749c7964e156cd5038fa3, entries=8, sequenceid=83, filesize=5.5 K 2024-11-24T18:45:44,662 DEBUG [M:0;f2b92657890a:43953 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2522b032cbe64ca8948f349a577570df as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2522b032cbe64ca8948f349a577570df 2024-11-24T18:45:44,664 INFO [M:0;f2b92657890a:43953 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2522b032cbe64ca8948f349a577570df, entries=9, sequenceid=83, filesize=6.4 K 2024-11-24T18:45:44,664 DEBUG [M:0;f2b92657890a:43953 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/47e18a20d91949559a78b7b9bc6d129c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/47e18a20d91949559a78b7b9bc6d129c 2024-11-24T18:45:44,665 INFO [M:0;f2b92657890a:43953 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 47e18a20d91949559a78b7b9bc6d129c 2024-11-24T18:45:44,666 INFO [M:0;f2b92657890a:43953 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cf68692-38ac-cefb-46c4-e6795680a854/dc0773f6-4fda-5fe9-bef6-5af1264196b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/47e18a20d91949559a78b7b9bc6d129c, entries=3, sequenceid=83, filesize=5.3 K 2024-11-24T18:45:44,666 INFO [M:0;f2b92657890a:43953 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=83, compaction requested=false 2024-11-24T18:45:44,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:44,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40227-0x1016e2a63a20001, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:44,667 INFO [RS:0;f2b92657890a:40227 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T18:45:44,667 INFO [RS:0;f2b92657890a:40227 {}] regionserver.HRegionServer(1031): Exiting; stopping=f2b92657890a,40227,1732473940146; zookeeper connection closed. 2024-11-24T18:45:44,667 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@710ee91c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@710ee91c 2024-11-24T18:45:44,668 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T18:45:44,668 INFO [M:0;f2b92657890a:43953 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T18:45:44,668 DEBUG [M:0;f2b92657890a:43953 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732473944598Disabling compacts and flushes for region at 1732473944598Disabling writes for close at 1732473944598Obtaining lock to block concurrent updates at 1732473944599 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732473944599Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1732473944599Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732473944599Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732473944600 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732473944619 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732473944620 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732473944622 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732473944638 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732473944638Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732473944640 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732473944657 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732473944657Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10b92640: reopening flushed file at 1732473944660 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2045fc29: reopening flushed file at 1732473944662 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@450b234c: reopening flushed file at 1732473944664 (+2 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=83, compaction requested=false at 1732473944666 (+2 ms)Writing region close event to WAL at 1732473944668 (+2 ms)Closed at 1732473944668 2024-11-24T18:45:44,669 INFO [M:0;f2b92657890a:43953 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T18:45:44,669 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T18:45:44,669 INFO [M:0;f2b92657890a:43953 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43953 2024-11-24T18:45:44,669 INFO [M:0;f2b92657890a:43953 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T18:45:44,782 INFO [M:0;f2b92657890a:43953 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T18:45:44,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:44,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43953-0x1016e2a63a20000, quorum=127.0.0.1:60450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T18:45:44,789 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T18:45:44,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T18:45:44,792 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T18:45:44,802 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 50) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=470 (was 502), ProcessCount=11 (was 11), AvailableMemoryMB=10253 (was 10381)