2024-11-12 14:53:11,743 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-12 14:53:11,773 main DEBUG Took 0.013720 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 14:53:11,774 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 14:53:11,774 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 14:53:11,775 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 14:53:11,777 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,786 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 14:53:11,799 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,801 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,802 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,803 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,804 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,806 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,807 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,807 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,808 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,809 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,810 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,811 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,811 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:53:11,811 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,812 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 14:53:11,813 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:53:11,815 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 14:53:11,816 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 14:53:11,817 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 14:53:11,818 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 14:53:11,819 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 14:53:11,826 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 14:53:11,829 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 14:53:11,832 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 14:53:11,832 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 14:53:11,833 main DEBUG createAppenders(={Console}) 2024-11-12 14:53:11,834 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-12 14:53:11,834 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-12 14:53:11,835 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-12 14:53:11,836 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 14:53:11,836 main DEBUG OutputStream closed 2024-11-12 14:53:11,837 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 14:53:11,837 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 14:53:11,838 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-12 14:53:11,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 14:53:11,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 14:53:11,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 14:53:11,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 14:53:11,948 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 14:53:11,948 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 14:53:11,949 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 14:53:11,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 14:53:11,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 14:53:11,953 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 14:53:11,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 14:53:11,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 14:53:11,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 14:53:11,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 14:53:11,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 14:53:11,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 14:53:11,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 14:53:11,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 14:53:11,961 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 14:53:11,962 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-12 14:53:11,962 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 14:53:11,964 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-12T14:53:11,983 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-12 14:53:11,987 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 14:53:11,988 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T14:53:12,347 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d 2024-11-12T14:53:12,379 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=277, ProcessCount=11, AvailableMemoryMB=8718 2024-11-12T14:53:12,521 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T14:53:12,673 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:12,700 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/cluster_2fc27376-ceb9-b5a1-eca5-28c0306244fd, deleteOnExit=true 2024-11-12T14:53:12,855 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/cluster_2fc27376-ceb9-b5a1-eca5-28c0306244fd/zookeeper_0, clientPort=51303, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/cluster_2fc27376-ceb9-b5a1-eca5-28c0306244fd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/cluster_2fc27376-ceb9-b5a1-eca5-28c0306244fd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T14:53:12,871 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51303 2024-11-12T14:53:12,872 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:13,002 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23 with version=8 2024-11-12T14:53:13,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/hbase-staging 2024-11-12T14:53:13,129 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T14:53:13,362 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:53:13,370 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:13,372 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:13,379 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:53:13,380 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:13,380 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:53:13,521 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T14:53:13,629 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:53:13,687 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T14:53:13,699 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T14:53:13,704 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:53:13,741 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 40476 (auto-detected) 2024-11-12T14:53:13,742 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T14:53:13,762 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40201 2024-11-12T14:53:13,807 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40201 connecting to ZooKeeper ensemble=127.0.0.1:51303 2024-11-12T14:53:13,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:402010x0, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:53:13,864 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40201-0x100346ca3550000 connected 2024-11-12T14:53:13,927 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:13,928 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:13,942 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:53:13,949 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23, hbase.cluster.distributed=false 2024-11-12T14:53:13,994 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:53:14,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40201 2024-11-12T14:53:14,033 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40201 2024-11-12T14:53:14,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40201 2024-11-12T14:53:14,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40201 2024-11-12T14:53:14,050 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40201 2024-11-12T14:53:14,217 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:53:14,220 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:14,222 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:14,222 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:53:14,223 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:14,224 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:53:14,231 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T14:53:14,246 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:53:14,248 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39415 2024-11-12T14:53:14,251 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39415 connecting to ZooKeeper ensemble=127.0.0.1:51303 2024-11-12T14:53:14,253 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:14,257 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:14,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:394150x0, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:53:14,274 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:394150x0, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:53:14,278 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39415-0x100346ca3550001 connected 2024-11-12T14:53:14,284 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T14:53:14,343 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/bucket.cache 2024-11-12T14:53:14,368 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T14:53:14,370 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T14:53:14,373 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T14:53:14,397 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T14:53:14,401 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T14:53:14,421 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T14:53:14,427 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T14:53:14,427 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/bucket.persistence is complete. 2024-11-12T14:53:14,431 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T14:53:14,436 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T14:53:14,456 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:53:14,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39415 2024-11-12T14:53:14,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39415 2024-11-12T14:53:14,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39415 2024-11-12T14:53:14,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39415 2024-11-12T14:53:14,485 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39415 2024-11-12T14:53:14,518 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:53:14,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:14,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:14,520 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:53:14,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:14,521 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:53:14,524 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T14:53:14,525 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:53:14,526 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44367 2024-11-12T14:53:14,529 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44367 connecting to ZooKeeper ensemble=127.0.0.1:51303 2024-11-12T14:53:14,530 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:14,533 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:14,540 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:443670x0, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:53:14,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:443670x0, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:53:14,541 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T14:53:14,549 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44367-0x100346ca3550002 connected 2024-11-12T14:53:14,557 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/bucket.cache 2024-11-12T14:53:14,558 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T14:53:14,558 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T14:53:14,558 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T14:53:14,559 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T14:53:14,559 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T14:53:14,560 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T14:53:14,560 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/bucket.persistence is complete. 2024-11-12T14:53:14,562 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T14:53:14,570 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T14:53:14,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T14:53:14,577 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:53:14,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44367 2024-11-12T14:53:14,585 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44367 2024-11-12T14:53:14,587 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44367 2024-11-12T14:53:14,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44367 2024-11-12T14:53:14,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44367 2024-11-12T14:53:14,613 DEBUG [M:0;3945e1ca6f47:40201 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3945e1ca6f47:40201 2024-11-12T14:53:14,615 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3945e1ca6f47,40201,1731423193207 2024-11-12T14:53:14,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:53:14,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:53:14,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:53:14,626 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3945e1ca6f47,40201,1731423193207 2024-11-12T14:53:14,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T14:53:14,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T14:53:14,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:14,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:14,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:14,658 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T14:53:14,660 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3945e1ca6f47,40201,1731423193207 from backup master directory 2024-11-12T14:53:14,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3945e1ca6f47,40201,1731423193207 2024-11-12T14:53:14,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:53:14,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:53:14,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:53:14,663 WARN [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:53:14,664 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3945e1ca6f47,40201,1731423193207 2024-11-12T14:53:14,666 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T14:53:14,668 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T14:53:14,677 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/hbase.id] with ID: 0c61201f-b1b7-4181-83a9-cb96524482b6 2024-11-12T14:53:14,677 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/.tmp/hbase.id 2024-11-12T14:53:14,682 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/hbase.id] 2024-11-12T14:53:14,745 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:14,754 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T14:53:14,790 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 33ms. 2024-11-12T14:53:14,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:14,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:14,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:14,860 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:53:14,864 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T14:53:14,931 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:53:14,939 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:53:15,030 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store 2024-11-12T14:53:15,069 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T14:53:15,076 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:15,079 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T14:53:15,079 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:15,080 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:15,097 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T14:53:15,098 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:15,098 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:15,099 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731423195078Disabling compacts and flushes for region at 1731423195078Disabling writes for close at 1731423195098 (+20 ms)Writing region close event to WAL at 1731423195098Closed at 1731423195098 2024-11-12T14:53:15,101 WARN [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/.initializing 2024-11-12T14:53:15,101 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/WALs/3945e1ca6f47,40201,1731423193207 2024-11-12T14:53:15,123 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:53:15,141 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C40201%2C1731423193207, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/WALs/3945e1ca6f47,40201,1731423193207, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/oldWALs, maxLogs=10 2024-11-12T14:53:15,194 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/WALs/3945e1ca6f47,40201,1731423193207/3945e1ca6f47%2C40201%2C1731423193207.1731423195149 2024-11-12T14:53:15,197 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T14:53:15,198 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:53:15,199 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:15,206 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:15,207 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:15,263 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:15,299 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T14:53:15,303 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:15,305 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:15,306 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:15,316 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T14:53:15,317 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:15,317 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:53:15,318 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:15,322 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T14:53:15,323 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:15,323 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:53:15,324 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:15,328 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T14:53:15,329 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:15,329 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:53:15,330 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:15,333 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:15,334 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:15,337 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:15,338 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:15,342 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:53:15,346 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:15,362 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:53:15,363 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72344840, jitterRate=0.07802212238311768}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:53:15,371 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731423195223Initializing all the Stores at 1731423195229 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423195230 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423195231 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423195231Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423195231Cleaning up temporary data from old regions at 1731423195338 (+107 ms)Region opened successfully at 1731423195371 (+33 ms) 2024-11-12T14:53:15,373 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T14:53:15,412 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1af41e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:53:15,451 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T14:53:15,470 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T14:53:15,470 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T14:53:15,475 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:53:15,477 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T14:53:15,486 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-12T14:53:15,487 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T14:53:15,535 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T14:53:15,549 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T14:53:15,554 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T14:53:15,558 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T14:53:15,561 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T14:53:15,563 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T14:53:15,566 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T14:53:15,571 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T14:53:15,574 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T14:53:15,577 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T14:53:15,579 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T14:53:15,616 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T14:53:15,618 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T14:53:15,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:53:15,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:53:15,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:53:15,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:15,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:15,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:15,646 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3945e1ca6f47,40201,1731423193207, sessionid=0x100346ca3550000, setting cluster-up flag (Was=false) 2024-11-12T14:53:15,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:15,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:15,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:15,679 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T14:53:15,682 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3945e1ca6f47,40201,1731423193207 2024-11-12T14:53:15,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:15,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:15,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:15,696 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T14:53:15,698 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3945e1ca6f47,40201,1731423193207 2024-11-12T14:53:15,708 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T14:53:15,796 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(746): ClusterId : 0c61201f-b1b7-4181-83a9-cb96524482b6 2024-11-12T14:53:15,796 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(746): ClusterId : 0c61201f-b1b7-4181-83a9-cb96524482b6 2024-11-12T14:53:15,799 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T14:53:15,800 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T14:53:15,805 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T14:53:15,805 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T14:53:15,805 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T14:53:15,805 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T14:53:15,810 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T14:53:15,811 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T14:53:15,811 DEBUG [RS:1;3945e1ca6f47:44367 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aac966f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:53:15,812 DEBUG [RS:0;3945e1ca6f47:39415 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a904f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:53:15,812 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:53:15,826 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T14:53:15,827 DEBUG [RS:1;3945e1ca6f47:44367 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;3945e1ca6f47:44367 2024-11-12T14:53:15,837 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T14:53:15,841 DEBUG [RS:0;3945e1ca6f47:39415 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3945e1ca6f47:39415 2024-11-12T14:53:15,847 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T14:53:15,848 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T14:53:15,848 DEBUG [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T14:53:15,849 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T14:53:15,849 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T14:53:15,849 DEBUG [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T14:53:15,848 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3945e1ca6f47,40201,1731423193207 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T14:53:15,853 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(2659): reportForDuty to master=3945e1ca6f47,40201,1731423193207 with port=39415, startcode=1731423194147 2024-11-12T14:53:15,856 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(2659): reportForDuty to master=3945e1ca6f47,40201,1731423193207 with port=44367, startcode=1731423194517 2024-11-12T14:53:15,862 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:53:15,862 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:53:15,862 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:53:15,865 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:53:15,865 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3945e1ca6f47:0, corePoolSize=10, maxPoolSize=10 2024-11-12T14:53:15,866 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:15,866 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:53:15,866 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:15,871 DEBUG [RS:1;3945e1ca6f47:44367 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T14:53:15,873 DEBUG [RS:0;3945e1ca6f47:39415 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T14:53:15,893 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:53:15,894 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T14:53:15,901 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731423225901 2024-11-12T14:53:15,904 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T14:53:15,906 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T14:53:15,906 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:15,906 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T14:53:15,911 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T14:53:15,912 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T14:53:15,912 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T14:53:15,912 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T14:53:15,934 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:15,944 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T14:53:15,945 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T14:53:15,946 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T14:53:15,973 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T14:53:15,974 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T14:53:15,981 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.large.0-1731423195975,5,FailOnTimeoutGroup] 2024-11-12T14:53:16,015 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.small.0-1731423195981,5,FailOnTimeoutGroup] 2024-11-12T14:53:16,015 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,016 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T14:53:16,019 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,020 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,021 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36623, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T14:53:16,022 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59601, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T14:53:16,031 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40201 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:16,035 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40201 {}] master.ServerManager(517): Registering regionserver=3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:16,043 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T14:53:16,044 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23 2024-11-12T14:53:16,048 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40201 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3945e1ca6f47,39415,1731423194147 2024-11-12T14:53:16,049 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40201 {}] master.ServerManager(517): Registering regionserver=3945e1ca6f47,39415,1731423194147 2024-11-12T14:53:16,053 DEBUG [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23 2024-11-12T14:53:16,053 DEBUG [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23 2024-11-12T14:53:16,053 DEBUG [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T14:53:16,053 DEBUG [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T14:53:16,053 DEBUG [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T14:53:16,053 DEBUG [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T14:53:16,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:53:16,060 DEBUG [RS:0;3945e1ca6f47:39415 {}] zookeeper.ZKUtil(111): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3945e1ca6f47,39415,1731423194147 2024-11-12T14:53:16,060 DEBUG [RS:1;3945e1ca6f47:44367 {}] zookeeper.ZKUtil(111): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:16,060 WARN [RS:0;3945e1ca6f47:39415 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:53:16,060 WARN [RS:1;3945e1ca6f47:44367 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:53:16,061 INFO [RS:0;3945e1ca6f47:39415 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:53:16,061 INFO [RS:1;3945e1ca6f47:44367 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:53:16,061 DEBUG [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,39415,1731423194147 2024-11-12T14:53:16,061 DEBUG [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:16,083 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3945e1ca6f47,44367,1731423194517] 2024-11-12T14:53:16,084 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3945e1ca6f47,39415,1731423194147] 2024-11-12T14:53:16,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:16,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T14:53:16,125 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T14:53:16,125 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T14:53:16,126 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T14:53:16,127 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:16,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:16,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T14:53:16,139 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T14:53:16,140 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:16,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:16,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T14:53:16,167 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T14:53:16,167 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:16,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:16,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T14:53:16,170 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T14:53:16,173 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T14:53:16,179 INFO [RS:1;3945e1ca6f47:44367 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:53:16,180 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,181 INFO [RS:0;3945e1ca6f47:39415 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:53:16,182 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,182 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T14:53:16,182 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T14:53:16,182 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:16,183 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:16,183 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T14:53:16,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740 2024-11-12T14:53:16,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740 2024-11-12T14:53:16,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T14:53:16,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T14:53:16,186 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:53:16,187 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T14:53:16,188 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T14:53:16,190 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,191 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,191 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,191 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,191 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,191 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,192 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:53:16,192 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,192 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,192 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,192 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,192 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,193 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,193 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:53:16,193 DEBUG [RS:1;3945e1ca6f47:44367 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:53:16,195 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T14:53:16,196 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T14:53:16,196 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,196 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,196 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,196 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,197 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,197 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,197 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:53:16,197 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,197 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,198 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,198 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,198 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,198 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:16,198 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:53:16,199 DEBUG [RS:0;3945e1ca6f47:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:53:16,205 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,205 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,205 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,206 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,206 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,207 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,39415,1731423194147-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:53:16,221 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,221 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,221 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,222 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,222 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,222 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,44367,1731423194517-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:53:16,227 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:53:16,228 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74412952, jitterRate=0.10883939266204834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:53:16,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731423196113Initializing all the Stores at 1731423196114 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423196114Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423196117 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423196117Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423196117Cleaning up temporary data from old regions at 1731423196186 (+69 ms)Region opened successfully at 1731423196230 (+44 ms) 2024-11-12T14:53:16,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T14:53:16,230 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T14:53:16,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T14:53:16,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T14:53:16,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T14:53:16,238 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T14:53:16,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731423196230Disabling compacts and flushes for region at 1731423196230Disabling writes for close at 1731423196231 (+1 ms)Writing region close event to WAL at 1731423196238 (+7 ms)Closed at 1731423196238 2024-11-12T14:53:16,241 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:53:16,241 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T14:53:16,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T14:53:16,252 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T14:53:16,254 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,39415,1731423194147-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,255 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,255 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.Replication(171): 3945e1ca6f47,39415,1731423194147 started 2024-11-12T14:53:16,261 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T14:53:16,261 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,44367,1731423194517-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,261 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:53:16,262 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,262 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.Replication(171): 3945e1ca6f47,44367,1731423194517 started 2024-11-12T14:53:16,264 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T14:53:16,276 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,277 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(1482): Serving as 3945e1ca6f47,39415,1731423194147, RpcServer on 3945e1ca6f47/172.17.0.2:39415, sessionid=0x100346ca3550001 2024-11-12T14:53:16,279 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T14:53:16,279 DEBUG [RS:0;3945e1ca6f47:39415 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3945e1ca6f47,39415,1731423194147 2024-11-12T14:53:16,280 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,39415,1731423194147' 2024-11-12T14:53:16,280 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T14:53:16,289 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T14:53:16,291 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T14:53:16,291 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T14:53:16,291 DEBUG [RS:0;3945e1ca6f47:39415 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3945e1ca6f47,39415,1731423194147 2024-11-12T14:53:16,291 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,39415,1731423194147' 2024-11-12T14:53:16,291 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T14:53:16,292 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T14:53:16,293 DEBUG [RS:0;3945e1ca6f47:39415 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T14:53:16,293 INFO [RS:0;3945e1ca6f47:39415 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T14:53:16,294 INFO [RS:0;3945e1ca6f47:39415 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T14:53:16,294 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:16,294 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(1482): Serving as 3945e1ca6f47,44367,1731423194517, RpcServer on 3945e1ca6f47/172.17.0.2:44367, sessionid=0x100346ca3550002 2024-11-12T14:53:16,295 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T14:53:16,295 DEBUG [RS:1;3945e1ca6f47:44367 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:16,295 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,44367,1731423194517' 2024-11-12T14:53:16,295 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T14:53:16,297 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T14:53:16,298 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T14:53:16,299 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T14:53:16,299 DEBUG [RS:1;3945e1ca6f47:44367 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:16,299 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,44367,1731423194517' 2024-11-12T14:53:16,299 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T14:53:16,300 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T14:53:16,302 DEBUG [RS:1;3945e1ca6f47:44367 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T14:53:16,302 INFO [RS:1;3945e1ca6f47:44367 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T14:53:16,302 INFO [RS:1;3945e1ca6f47:44367 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T14:53:16,400 INFO [RS:0;3945e1ca6f47:39415 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:53:16,402 INFO [RS:0;3945e1ca6f47:39415 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3945e1ca6f47%2C39415%2C1731423194147, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,39415,1731423194147, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs, maxLogs=55 2024-11-12T14:53:16,403 INFO [RS:1;3945e1ca6f47:44367 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:53:16,405 INFO [RS:1;3945e1ca6f47:44367 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3945e1ca6f47%2C44367%2C1731423194517, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,44367,1731423194517, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs, maxLogs=55 2024-11-12T14:53:16,416 WARN [3945e1ca6f47:40201 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T14:53:16,453 INFO [RS:0;3945e1ca6f47:39415 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,39415,1731423194147/3945e1ca6f47%2C39415%2C1731423194147.1731423196405 2024-11-12T14:53:16,456 INFO [RS:1;3945e1ca6f47:44367 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,44367,1731423194517/3945e1ca6f47%2C44367%2C1731423194517.1731423196408 2024-11-12T14:53:16,458 DEBUG [RS:0;3945e1ca6f47:39415 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T14:53:16,458 DEBUG [RS:1;3945e1ca6f47:44367 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T14:53:16,675 DEBUG [3945e1ca6f47:40201 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-12T14:53:16,699 DEBUG [3945e1ca6f47:40201 {}] balancer.BalancerClusterState(204): Hosts are {3945e1ca6f47=0} racks are {/default-rack=0} 2024-11-12T14:53:16,706 DEBUG [3945e1ca6f47:40201 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T14:53:16,707 DEBUG [3945e1ca6f47:40201 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T14:53:16,707 DEBUG [3945e1ca6f47:40201 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T14:53:16,707 DEBUG [3945e1ca6f47:40201 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T14:53:16,707 INFO [3945e1ca6f47:40201 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T14:53:16,707 INFO [3945e1ca6f47:40201 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T14:53:16,707 DEBUG [3945e1ca6f47:40201 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T14:53:16,717 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:16,725 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,44367,1731423194517, state=OPENING 2024-11-12T14:53:16,748 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T14:53:16,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:16,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:16,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:16,752 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:16,753 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:16,754 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:16,757 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:53:16,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,44367,1731423194517}] 2024-11-12T14:53:16,957 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:53:16,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56659, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:53:16,993 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T14:53:16,993 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:53:16,995 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T14:53:16,996 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3945e1ca6f47%2C44367%2C1731423194517.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,44367,1731423194517, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs, maxLogs=55 2024-11-12T14:53:17,045 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,44367,1731423194517/3945e1ca6f47%2C44367%2C1731423194517.meta.1731423196999.meta 2024-11-12T14:53:17,047 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T14:53:17,048 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:53:17,051 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T14:53:17,054 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T14:53:17,063 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T14:53:17,068 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T14:53:17,069 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:17,070 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T14:53:17,070 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T14:53:17,073 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T14:53:17,075 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T14:53:17,075 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:17,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:17,076 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T14:53:17,076 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T14:53:17,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:17,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:17,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T14:53:17,078 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T14:53:17,079 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:17,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:17,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T14:53:17,080 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T14:53:17,081 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:17,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:17,081 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T14:53:17,083 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740 2024-11-12T14:53:17,084 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740 2024-11-12T14:53:17,085 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T14:53:17,085 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T14:53:17,085 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:53:17,087 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T14:53:17,087 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61129090, jitterRate=-0.08910557627677917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:53:17,088 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T14:53:17,090 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731423197071Writing region info on filesystem at 1731423197071Initializing all the Stores at 1731423197071Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423197071Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423197073 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423197073Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423197073Cleaning up temporary data from old regions at 1731423197085 (+12 ms)Running coprocessor post-open hooks at 1731423197088 (+3 ms)Region opened successfully at 1731423197090 (+2 ms) 2024-11-12T14:53:17,097 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731423196946 2024-11-12T14:53:17,112 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:17,114 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,44367,1731423194517, state=OPEN 2024-11-12T14:53:17,114 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T14:53:17,115 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T14:53:17,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:53:17,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:53:17,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:17,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:17,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:53:17,117 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:17,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:17,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T14:53:17,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,44367,1731423194517 in 357 msec 2024-11-12T14:53:17,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T14:53:17,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 874 msec 2024-11-12T14:53:17,132 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:53:17,132 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T14:53:17,160 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:53:17,161 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,44367,1731423194517, seqNum=-1] 2024-11-12T14:53:17,193 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:53:17,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47845, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:53:17,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4730 sec 2024-11-12T14:53:17,225 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731423197225, completionTime=-1 2024-11-12T14:53:17,229 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-12T14:53:17,229 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:53:17,268 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T14:53:17,269 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731423257268 2024-11-12T14:53:17,269 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731423317269 2024-11-12T14:53:17,269 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 39 msec 2024-11-12T14:53:17,272 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:53:17,280 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,40201,1731423193207-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:17,281 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,40201,1731423193207-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:17,281 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,40201,1731423193207-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:17,283 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3945e1ca6f47:40201, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:17,283 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:17,286 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:17,292 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T14:53:17,330 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.664sec 2024-11-12T14:53:17,336 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T14:53:17,338 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T14:53:17,339 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T14:53:17,340 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T14:53:17,343 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T14:53:17,344 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,40201,1731423193207-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:53:17,345 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,40201,1731423193207-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T14:53:17,349 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T14:53:17,350 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T14:53:17,351 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,40201,1731423193207-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:17,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752a9183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:53:17,411 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T14:53:17,412 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T14:53:17,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3945e1ca6f47,40201,-1 for getting cluster id 2024-11-12T14:53:17,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:53:17,459 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0c61201f-b1b7-4181-83a9-cb96524482b6' 2024-11-12T14:53:17,468 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:53:17,468 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0c61201f-b1b7-4181-83a9-cb96524482b6" 2024-11-12T14:53:17,471 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db3b6a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:53:17,471 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3945e1ca6f47,40201,-1] 2024-11-12T14:53:17,474 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:53:17,476 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:17,478 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:53:17,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a5538f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:53:17,481 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:53:17,489 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,44367,1731423194517, seqNum=-1] 2024-11-12T14:53:17,489 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:53:17,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:53:17,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3945e1ca6f47,40201,1731423193207 2024-11-12T14:53:17,514 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T14:53:17,518 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3945e1ca6f47,40201,1731423193207 2024-11-12T14:53:17,520 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32209e54 2024-11-12T14:53:17,521 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T14:53:17,524 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T14:53:17,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40201 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731423197510', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:53:17,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40201 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731423197510 2024-11-12T14:53:17,537 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731423197510 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T14:53:17,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40201 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731423197510" procId is: 4 2024-11-12T14:53:17,539 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:17,541 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731423197510 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T14:53:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40201 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:53:17,553 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731423197510-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 176f46e43dc4e17c65f19262a0c4de42, NAME => 'testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731423197510', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23 2024-11-12T14:53:17,560 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731423197510-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:17,561 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731423197510-pool-0 {}] regionserver.HRegion(1722): Closing 176f46e43dc4e17c65f19262a0c4de42, disabling compactions & flushes 2024-11-12T14:53:17,561 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731423197510-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:17,561 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731423197510-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:17,561 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731423197510-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. after waiting 0 ms 2024-11-12T14:53:17,561 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731423197510-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:17,561 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731423197510-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:17,561 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731423197510-pool-0 {}] regionserver.HRegion(1676): Region close journal for 176f46e43dc4e17c65f19262a0c4de42: Waiting for close lock at 1731423197561Disabling compacts and flushes for region at 1731423197561Disabling writes for close at 1731423197561Writing region close event to WAL at 1731423197561Closed at 1731423197561 2024-11-12T14:53:17,563 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731423197510 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T14:53:17,567 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731423197563"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731423197563"}]},"ts":"1731423197563"} 2024-11-12T14:53:17,572 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T14:53:17,573 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731423197510 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T14:53:17,576 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731423197510","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731423197573"}]},"ts":"1731423197573"} 2024-11-12T14:53:17,580 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731423197510, state=ENABLING in hbase:meta 2024-11-12T14:53:17,580 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {3945e1ca6f47=0} racks are {/default-rack=0} 2024-11-12T14:53:17,581 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T14:53:17,581 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T14:53:17,581 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T14:53:17,582 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T14:53:17,582 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T14:53:17,582 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T14:53:17,582 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T14:53:17,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731423197510, region=176f46e43dc4e17c65f19262a0c4de42, ASSIGN}] 2024-11-12T14:53:17,585 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731423197510, region=176f46e43dc4e17c65f19262a0c4de42, ASSIGN 2024-11-12T14:53:17,586 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731423197510, region=176f46e43dc4e17c65f19262a0c4de42, ASSIGN; state=OFFLINE, location=3945e1ca6f47,44367,1731423194517; forceNewPlan=false, retain=false 2024-11-12T14:53:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40201 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:53:17,739 INFO [3945e1ca6f47:40201 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T14:53:17,740 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=176f46e43dc4e17c65f19262a0c4de42, regionState=OPENING, regionLocation=3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:17,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731423197510, region=176f46e43dc4e17c65f19262a0c4de42, ASSIGN because future has completed 2024-11-12T14:53:17,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 176f46e43dc4e17c65f19262a0c4de42, server=3945e1ca6f47,44367,1731423194517}] 2024-11-12T14:53:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40201 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:53:17,902 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:17,902 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 176f46e43dc4e17c65f19262a0c4de42, NAME => 'testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:53:17,903 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731423197510 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:17,903 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:17,903 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:17,903 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:17,905 INFO [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:17,911 INFO [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 176f46e43dc4e17c65f19262a0c4de42 columnFamilyName family 2024-11-12T14:53:17,911 DEBUG [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:17,912 INFO [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] regionserver.HStore(327): Store=176f46e43dc4e17c65f19262a0c4de42/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:53:17,912 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:17,915 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:17,915 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:17,915 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:17,915 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:17,917 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:17,923 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:53:17,923 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 176f46e43dc4e17c65f19262a0c4de42; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71896669, jitterRate=0.07134385406970978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:53:17,924 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:17,925 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 176f46e43dc4e17c65f19262a0c4de42: Running coprocessor pre-open hook at 1731423197904Writing region info on filesystem at 1731423197904Initializing all the Stores at 1731423197905 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423197905Cleaning up temporary data from old regions at 1731423197916 (+11 ms)Running coprocessor post-open hooks at 1731423197924 (+8 ms)Region opened successfully at 1731423197925 (+1 ms) 2024-11-12T14:53:17,926 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42., pid=6, masterSystemTime=1731423197898 2024-11-12T14:53:17,929 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:17,929 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:17,929 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=176f46e43dc4e17c65f19262a0c4de42, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:17,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 176f46e43dc4e17c65f19262a0c4de42, server=3945e1ca6f47,44367,1731423194517 because future has completed 2024-11-12T14:53:17,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T14:53:17,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 176f46e43dc4e17c65f19262a0c4de42, server=3945e1ca6f47,44367,1731423194517 in 188 msec 2024-11-12T14:53:17,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T14:53:17,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731423197510, region=176f46e43dc4e17c65f19262a0c4de42, ASSIGN in 352 msec 2024-11-12T14:53:17,937 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731423197510 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T14:53:17,938 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731423197510","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731423197937"}]},"ts":"1731423197937"} 2024-11-12T14:53:17,940 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731423197510, state=ENABLED in hbase:meta 2024-11-12T14:53:17,941 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731423197510 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T14:53:17,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731423197510 in 410 msec 2024-11-12T14:53:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40201 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:53:18,178 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731423197510 completed 2024-11-12T14:53:18,178 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731423197510 get assigned. Timeout = 60000ms 2024-11-12T14:53:18,181 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T14:53:18,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731423197510 assigned to meta. Checking AM states. 2024-11-12T14:53:18,188 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T14:53:18,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731423197510 assigned. 2024-11-12T14:53:18,199 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731423197510', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42., hostname=3945e1ca6f47,44367,1731423194517, seqNum=2] 2024-11-12T14:53:18,212 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 176f46e43dc4e17c65f19262a0c4de42 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-12T14:53:18,244 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dc564508b63242f59eba832efd567204 2024-11-12T14:53:18,295 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/.tmp/family/dc564508b63242f59eba832efd567204 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-12T14:53:18,301 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/.tmp/family/dc564508b63242f59eba832efd567204 2024-11-12T14:53:18,336 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/.tmp/family/dc564508b63242f59eba832efd567204 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/family/dc564508b63242f59eba832efd567204 2024-11-12T14:53:18,339 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/family/dc564508b63242f59eba832efd567204, delay=944 ms 2024-11-12T14:53:18,341 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/family/dc564508b63242f59eba832efd567204, entries=2, sequenceid=6, filesize=4.9 K 2024-11-12T14:53:18,349 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 176f46e43dc4e17c65f19262a0c4de42 in 136ms, sequenceid=6, compaction requested=false 2024-11-12T14:53:18,350 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731423197510' 2024-11-12T14:53:18,353 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 176f46e43dc4e17c65f19262a0c4de42: 2024-11-12T14:53:19,287 DEBUG [hfile-prefetch-1731423198339 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key dc564508b63242f59eba832efd567204_0. 2024-11-12T14:53:19,294 DEBUG [hfile-prefetch-1731423198339 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key dc564508b63242f59eba832efd567204_117. 2024-11-12T14:53:19,296 DEBUG [hfile-prefetch-1731423198339 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/family/dc564508b63242f59eba832efd567204, with total blocks 2, and data blocks 1 2024-11-12T14:53:19,296 DEBUG [hfile-prefetch-1731423198339 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-12T14:53:19,296 DEBUG [hfile-prefetch-1731423198339 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/family/dc564508b63242f59eba832efd567204, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-12T14:53:19,297 DEBUG [hfile-prefetch-1731423198339 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/family/dc564508b63242f59eba832efd567204. Will sleep 100ms and try the verification again. 2024-11-12T14:53:19,308 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-12T14:53:19,308 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-12T14:53:19,308 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T14:53:19,308 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T14:53:19,322 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T14:53:19,323 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731423199308 2024-11-12T14:53:19,323 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-12T14:53:19,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40201 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=176f46e43dc4e17c65f19262a0c4de42, source=3945e1ca6f47,44367,1731423194517, destination=3945e1ca6f47,39415,1731423194147, warming up region on 3945e1ca6f47,39415,1731423194147 2024-11-12T14:53:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40201 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:53:19,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40201 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=176f46e43dc4e17c65f19262a0c4de42, source=3945e1ca6f47,44367,1731423194517, destination=3945e1ca6f47,39415,1731423194147, running balancer 2024-11-12T14:53:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40201 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731423197510, region=176f46e43dc4e17c65f19262a0c4de42, REOPEN/MOVE 2024-11-12T14:53:19,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35723, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:53:19,374 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731423197510, region=176f46e43dc4e17c65f19262a0c4de42, REOPEN/MOVE 2024-11-12T14:53:19,375 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=176f46e43dc4e17c65f19262a0c4de42, regionState=CLOSING, regionLocation=3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:19,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(7855): Warmup {ENCODED => 176f46e43dc4e17c65f19262a0c4de42, NAME => 'testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:53:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:19,377 INFO [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,378 INFO [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 176f46e43dc4e17c65f19262a0c4de42 columnFamilyName family 2024-11-12T14:53:19,378 DEBUG [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:19,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731423197510, region=176f46e43dc4e17c65f19262a0c4de42, REOPEN/MOVE because future has completed 2024-11-12T14:53:19,379 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-12T14:53:19,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 176f46e43dc4e17c65f19262a0c4de42, server=3945e1ca6f47,44367,1731423194517}] 2024-11-12T14:53:19,382 DEBUG [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/family/dc564508b63242f59eba832efd567204 2024-11-12T14:53:19,383 INFO [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] regionserver.HStore(327): Store=176f46e43dc4e17c65f19262a0c4de42/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:53:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(1722): Closing 176f46e43dc4e17c65f19262a0c4de42, disabling compactions & flushes 2024-11-12T14:53:19,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. after waiting 0 ms 2024-11-12T14:53:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,384 DEBUG [StoreFileCloser-176f46e43dc4e17c65f19262a0c4de42-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/family/dc564508b63242f59eba832efd567204 2024-11-12T14:53:19,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(1676): Region close journal for 176f46e43dc4e17c65f19262a0c4de42: Waiting for close lock at 1731423199383Disabling compacts and flushes for region at 1731423199383Disabling writes for close at 1731423199383Writing region close event to WAL at 1731423199385 (+2 ms)Closed at 1731423199385 2024-11-12T14:53:19,387 WARN [hfile-prefetch-1731423198339 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/family/dc564508b63242f59eba832efd567204, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-12T14:53:19,388 DEBUG [hfile-prefetch-1731423198339 {}] hfile.PrefetchExecutor(121): Prefetch completed for dc564508b63242f59eba832efd567204 2024-11-12T14:53:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40201 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-12T14:53:19,537 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,538 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-12T14:53:19,538 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 176f46e43dc4e17c65f19262a0c4de42, disabling compactions & flushes 2024-11-12T14:53:19,538 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,538 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,539 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. after waiting 0 ms 2024-11-12T14:53:19,539 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,540 DEBUG [StoreFileCloser-176f46e43dc4e17c65f19262a0c4de42-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file dc564508b63242f59eba832efd567204, starting offset: 0, end offset: 9223372036854775807 2024-11-12T14:53:19,542 DEBUG [StoreFileCloser-176f46e43dc4e17c65f19262a0c4de42-family-1 {}] bucket.BucketCache(857): removed key dc564508b63242f59eba832efd567204_0 from back map with offset lock 2094080 in the evict process 2024-11-12T14:53:19,547 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-12T14:53:19,549 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,549 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 176f46e43dc4e17c65f19262a0c4de42: Waiting for close lock at 1731423199538Running coprocessor pre-close hooks at 1731423199538Disabling compacts and flushes for region at 1731423199538Disabling writes for close at 1731423199539 (+1 ms)Writing region close event to WAL at 1731423199542 (+3 ms)Running coprocessor post-close hooks at 1731423199548 (+6 ms)Closed at 1731423199549 (+1 ms) 2024-11-12T14:53:19,549 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 176f46e43dc4e17c65f19262a0c4de42 move to 3945e1ca6f47,39415,1731423194147 record at close sequenceid=6 2024-11-12T14:53:19,554 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,554 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=176f46e43dc4e17c65f19262a0c4de42, regionState=CLOSED 2024-11-12T14:53:19,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 176f46e43dc4e17c65f19262a0c4de42, server=3945e1ca6f47,44367,1731423194517 because future has completed 2024-11-12T14:53:19,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T14:53:19,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 176f46e43dc4e17c65f19262a0c4de42, server=3945e1ca6f47,44367,1731423194517 in 178 msec 2024-11-12T14:53:19,561 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731423197510, region=176f46e43dc4e17c65f19262a0c4de42, REOPEN/MOVE; state=CLOSED, location=3945e1ca6f47,39415,1731423194147; forceNewPlan=false, retain=false 2024-11-12T14:53:19,565 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-12T14:53:19,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-12T14:53:19,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T14:53:19,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T14:53:19,568 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T14:53:19,568 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731423199565 2024-11-12T14:53:19,568 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-12T14:53:19,712 INFO [3945e1ca6f47:40201 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T14:53:19,713 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=176f46e43dc4e17c65f19262a0c4de42, regionState=OPENING, regionLocation=3945e1ca6f47,39415,1731423194147 2024-11-12T14:53:19,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731423197510, region=176f46e43dc4e17c65f19262a0c4de42, REOPEN/MOVE because future has completed 2024-11-12T14:53:19,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 176f46e43dc4e17c65f19262a0c4de42, server=3945e1ca6f47,39415,1731423194147}] 2024-11-12T14:53:19,879 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,879 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 176f46e43dc4e17c65f19262a0c4de42, NAME => 'testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:53:19,880 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731423197510 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,880 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:19,881 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,881 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,883 INFO [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,885 INFO [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 176f46e43dc4e17c65f19262a0c4de42 columnFamilyName family 2024-11-12T14:53:19,886 DEBUG [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:19,888 DEBUG [StoreFileOpener-176f46e43dc4e17c65f19262a0c4de42-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/family/dc564508b63242f59eba832efd567204, delay=952 ms 2024-11-12T14:53:19,890 DEBUG [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/family/dc564508b63242f59eba832efd567204 2024-11-12T14:53:19,890 INFO [StoreOpener-176f46e43dc4e17c65f19262a0c4de42-1 {}] regionserver.HStore(327): Store=176f46e43dc4e17c65f19262a0c4de42/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:53:19,890 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,891 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,892 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,892 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,892 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,897 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,898 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 176f46e43dc4e17c65f19262a0c4de42; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71882860, jitterRate=0.07113808393478394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:53:19,898 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,900 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 176f46e43dc4e17c65f19262a0c4de42: Running coprocessor pre-open hook at 1731423199882Writing region info on filesystem at 1731423199882Initializing all the Stores at 1731423199882Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423199883 (+1 ms)Cleaning up temporary data from old regions at 1731423199892 (+9 ms)Running coprocessor post-open hooks at 1731423199898 (+6 ms)Region opened successfully at 1731423199899 (+1 ms) 2024-11-12T14:53:19,900 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42., pid=9, masterSystemTime=1731423199874 2024-11-12T14:53:19,903 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,904 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,904 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=176f46e43dc4e17c65f19262a0c4de42, regionState=OPEN, openSeqNum=10, regionLocation=3945e1ca6f47,39415,1731423194147 2024-11-12T14:53:19,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 176f46e43dc4e17c65f19262a0c4de42, server=3945e1ca6f47,39415,1731423194147 because future has completed 2024-11-12T14:53:19,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-12T14:53:19,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 176f46e43dc4e17c65f19262a0c4de42, server=3945e1ca6f47,39415,1731423194147 in 188 msec 2024-11-12T14:53:19,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731423197510, region=176f46e43dc4e17c65f19262a0c4de42, REOPEN/MOVE in 539 msec 2024-11-12T14:53:19,927 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T14:53:19,927 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:53:19,927 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:53:19,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:19,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:19,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:53:19,933 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T14:53:19,933 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-12T14:53:19,934 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3945e1ca6f47,40201,1731423193207 2024-11-12T14:53:19,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:53:19,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:53:19,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:53:19,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:19,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:19,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:19,936 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:53:19,936 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:53:19,937 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:53:19,937 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:53:19,937 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:53:19,937 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:53:19,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:19,938 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3945e1ca6f47,39415,1731423194147' ***** 2024-11-12T14:53:19,938 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T14:53:19,938 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3945e1ca6f47,44367,1731423194517' ***** 2024-11-12T14:53:19,938 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T14:53:19,938 INFO [RS:0;3945e1ca6f47:39415 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T14:53:19,938 INFO [RS:1;3945e1ca6f47:44367 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T14:53:19,939 INFO [RS:1;3945e1ca6f47:44367 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/bucket.persistence 2024-11-12T14:53:19,939 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T14:53:19,939 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T14:53:19,939 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T14:53:19,939 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T14:53:19,939 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T14:53:19,939 INFO [RS:1;3945e1ca6f47:44367 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T14:53:19,939 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T14:53:19,939 INFO [RS:0;3945e1ca6f47:39415 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/bucket.persistence 2024-11-12T14:53:19,939 INFO [RS:0;3945e1ca6f47:39415 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T14:53:19,939 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T14:53:19,939 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T14:53:19,949 DEBUG [RS:1;3945e1ca6f47:44367 {}] bucket.BucketCache(1488): Thread RS:1;3945e1ca6f47:44367 started persisting bucket cache to file 2024-11-12T14:53:19,949 DEBUG [RS:1;3945e1ca6f47:44367 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T14:53:19,949 DEBUG [RS:0;3945e1ca6f47:39415 {}] bucket.BucketCache(1488): Thread RS:0;3945e1ca6f47:39415 started persisting bucket cache to file 2024-11-12T14:53:19,949 DEBUG [RS:1;3945e1ca6f47:44367 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T14:53:19,949 DEBUG [RS:0;3945e1ca6f47:39415 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T14:53:19,949 DEBUG [RS:0;3945e1ca6f47:39415 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T14:53:19,951 INFO [regionserver/3945e1ca6f47:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:53:19,951 INFO [regionserver/3945e1ca6f47:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:53:19,953 DEBUG [RS:1;3945e1ca6f47:44367 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T14:53:19,953 DEBUG [RS:1;3945e1ca6f47:44367 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731423199949 2024-11-12T14:53:19,953 DEBUG [RS:0;3945e1ca6f47:39415 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T14:53:19,953 DEBUG [RS:0;3945e1ca6f47:39415 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731423199949 2024-11-12T14:53:19,953 DEBUG [RS:1;3945e1ca6f47:44367 {}] bucket.BucketCache(1510): Thread RS:1;3945e1ca6f47:44367 finished persisting bucket cache to file, renaming 2024-11-12T14:53:19,953 DEBUG [RS:0;3945e1ca6f47:39415 {}] bucket.BucketCache(1510): Thread RS:0;3945e1ca6f47:39415 finished persisting bucket cache to file, renaming 2024-11-12T14:53:19,953 WARN [RS:0;3945e1ca6f47:39415 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-12T14:53:19,954 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T14:53:19,954 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T14:53:19,954 INFO [RS:1;3945e1ca6f47:44367 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T14:53:19,954 INFO [RS:0;3945e1ca6f47:39415 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T14:53:19,954 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T14:53:19,954 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T14:53:19,954 INFO [RS:0;3945e1ca6f47:39415 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T14:53:19,954 INFO [RS:1;3945e1ca6f47:44367 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T14:53:19,954 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(959): stopping server 3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:19,954 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(3091): Received CLOSE for 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,954 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:53:19,954 INFO [RS:1;3945e1ca6f47:44367 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;3945e1ca6f47:44367. 2024-11-12T14:53:19,955 DEBUG [RS:1;3945e1ca6f47:44367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:53:19,955 DEBUG [RS:1;3945e1ca6f47:44367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:19,955 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(959): stopping server 3945e1ca6f47,39415,1731423194147 2024-11-12T14:53:19,955 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T14:53:19,955 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:53:19,955 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T14:53:19,955 INFO [RS:0;3945e1ca6f47:39415 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3945e1ca6f47:39415. 2024-11-12T14:53:19,955 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T14:53:19,955 DEBUG [RS:0;3945e1ca6f47:39415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:53:19,955 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T14:53:19,955 DEBUG [RS:0;3945e1ca6f47:39415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:19,955 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T14:53:19,955 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T14:53:19,956 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T14:53:19,956 DEBUG [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(1325): Online Regions={176f46e43dc4e17c65f19262a0c4de42=testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42.} 2024-11-12T14:53:19,956 DEBUG [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T14:53:19,956 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T14:53:19,956 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 176f46e43dc4e17c65f19262a0c4de42, disabling compactions & flushes 2024-11-12T14:53:19,956 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T14:53:19,956 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,956 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T14:53:19,956 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,956 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T14:53:19,956 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. after waiting 0 ms 2024-11-12T14:53:19,956 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T14:53:19,956 DEBUG [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(1351): Waiting on 176f46e43dc4e17c65f19262a0c4de42 2024-11-12T14:53:19,956 DEBUG [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T14:53:19,956 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,956 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T14:53:19,956 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-12T14:53:19,957 DEBUG [StoreFileCloser-176f46e43dc4e17c65f19262a0c4de42-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/family/dc564508b63242f59eba832efd567204 2024-11-12T14:53:19,961 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnRegionMove1731423197510/176f46e43dc4e17c65f19262a0c4de42/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-12T14:53:19,961 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,961 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 176f46e43dc4e17c65f19262a0c4de42: Waiting for close lock at 1731423199955Running coprocessor pre-close hooks at 1731423199956 (+1 ms)Disabling compacts and flushes for region at 1731423199956Disabling writes for close at 1731423199956Writing region close event to WAL at 1731423199957 (+1 ms)Running coprocessor post-close hooks at 1731423199961 (+4 ms)Closed at 1731423199961 2024-11-12T14:53:19,961 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42. 2024-11-12T14:53:19,968 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2345f03d33ba41a782bf2ce3896620d3 2024-11-12T14:53:19,983 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/info/2345f03d33ba41a782bf2ce3896620d3 is 205, key is testBlockEvictionOnRegionMove1731423197510,,1731423197525.176f46e43dc4e17c65f19262a0c4de42./info:regioninfo/1731423199904/Put/seqid=0 2024-11-12T14:53:19,984 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/info/2345f03d33ba41a782bf2ce3896620d3 2024-11-12T14:53:19,992 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fb22efc9b1574ac08c6666e875d7845b 2024-11-12T14:53:20,004 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/ns/fb22efc9b1574ac08c6666e875d7845b is 43, key is default/ns:d/1731423197203/Put/seqid=0 2024-11-12T14:53:20,004 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/ns/fb22efc9b1574ac08c6666e875d7845b 2024-11-12T14:53:20,013 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4f27b658d43441689a98ed4d485f7a98 2024-11-12T14:53:20,032 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/table/4f27b658d43441689a98ed4d485f7a98 is 78, key is testBlockEvictionOnRegionMove1731423197510/table:state/1731423197937/Put/seqid=0 2024-11-12T14:53:20,033 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/table/4f27b658d43441689a98ed4d485f7a98 2024-11-12T14:53:20,036 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/info/2345f03d33ba41a782bf2ce3896620d3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/2345f03d33ba41a782bf2ce3896620d3 2024-11-12T14:53:20,037 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/2345f03d33ba41a782bf2ce3896620d3, delay=936 ms 2024-11-12T14:53:20,038 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/2345f03d33ba41a782bf2ce3896620d3, entries=15, sequenceid=15, filesize=7.6 K 2024-11-12T14:53:20,039 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/ns/fb22efc9b1574ac08c6666e875d7845b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/ns/fb22efc9b1574ac08c6666e875d7845b 2024-11-12T14:53:20,040 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/ns/fb22efc9b1574ac08c6666e875d7845b, delay=923 ms 2024-11-12T14:53:20,041 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/ns/fb22efc9b1574ac08c6666e875d7845b, entries=2, sequenceid=15, filesize=5.0 K 2024-11-12T14:53:20,042 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/table/4f27b658d43441689a98ed4d485f7a98 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/table/4f27b658d43441689a98ed4d485f7a98 2024-11-12T14:53:20,043 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/table/4f27b658d43441689a98ed4d485f7a98, delay=918 ms 2024-11-12T14:53:20,044 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/table/4f27b658d43441689a98ed4d485f7a98, entries=2, sequenceid=15, filesize=5.3 K 2024-11-12T14:53:20,045 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false 2024-11-12T14:53:20,045 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T14:53:20,047 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/2345f03d33ba41a782bf2ce3896620d3 2024-11-12T14:53:20,047 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/ns/fb22efc9b1574ac08c6666e875d7845b 2024-11-12T14:53:20,049 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/table/4f27b658d43441689a98ed4d485f7a98 2024-11-12T14:53:20,053 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-12T14:53:20,054 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T14:53:20,054 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T14:53:20,055 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731423199956Running coprocessor pre-close hooks at 1731423199956Disabling compacts and flushes for region at 1731423199956Disabling writes for close at 1731423199956Obtaining lock to block concurrent updates at 1731423199956Preparing flush snapshotting stores in 1588230740 at 1731423199956Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731423199957 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731423199957Flushing 1588230740/info: creating writer at 1731423199957Flushing 1588230740/info: appending metadata at 1731423199983 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731423199983Flushing 1588230740/ns: creating writer at 1731423199986 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731423200003 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731423200003Flushing 1588230740/table: creating writer at 1731423200007 (+4 ms)Flushing 1588230740/table: appending metadata at 1731423200032 (+25 ms)Flushing 1588230740/table: closing flushed file at 1731423200032Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26542c9b: reopening flushed file at 1731423200035 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b47c4aa: reopening flushed file at 1731423200038 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@370f3428: reopening flushed file at 1731423200041 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false at 1731423200045 (+4 ms)Writing region close event to WAL at 1731423200049 (+4 ms)Running coprocessor post-close hooks at 1731423200054 (+5 ms)Closed at 1731423200054 2024-11-12T14:53:20,055 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T14:53:20,156 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(976): stopping server 3945e1ca6f47,44367,1731423194517; all regions closed. 2024-11-12T14:53:20,156 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(976): stopping server 3945e1ca6f47,39415,1731423194147; all regions closed. 2024-11-12T14:53:20,164 DEBUG [RS:1;3945e1ca6f47:44367 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs 2024-11-12T14:53:20,164 DEBUG [RS:0;3945e1ca6f47:39415 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs 2024-11-12T14:53:20,164 INFO [RS:1;3945e1ca6f47:44367 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C44367%2C1731423194517.meta:.meta(num 1731423196999) 2024-11-12T14:53:20,164 INFO [RS:0;3945e1ca6f47:39415 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C39415%2C1731423194147:(num 1731423196405) 2024-11-12T14:53:20,164 DEBUG [RS:0;3945e1ca6f47:39415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:20,165 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:53:20,165 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:53:20,165 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.ChoreService(370): Chore service for: regionserver/3945e1ca6f47:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T14:53:20,166 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T14:53:20,166 INFO [regionserver/3945e1ca6f47:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:53:20,166 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T14:53:20,166 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T14:53:20,166 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:53:20,167 INFO [RS:0;3945e1ca6f47:39415 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39415 2024-11-12T14:53:20,167 DEBUG [RS:1;3945e1ca6f47:44367 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs 2024-11-12T14:53:20,168 INFO [RS:1;3945e1ca6f47:44367 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C44367%2C1731423194517:(num 1731423196408) 2024-11-12T14:53:20,168 DEBUG [RS:1;3945e1ca6f47:44367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:20,168 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:53:20,168 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:53:20,168 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.ChoreService(370): Chore service for: regionserver/3945e1ca6f47:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T14:53:20,169 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:53:20,169 INFO [regionserver/3945e1ca6f47:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:53:20,169 INFO [RS:1;3945e1ca6f47:44367 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44367 2024-11-12T14:53:20,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3945e1ca6f47,39415,1731423194147 2024-11-12T14:53:20,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:53:20,172 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:53:20,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3945e1ca6f47,44367,1731423194517 2024-11-12T14:53:20,172 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:53:20,174 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3945e1ca6f47,44367,1731423194517] 2024-11-12T14:53:20,175 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3945e1ca6f47,44367,1731423194517 already deleted, retry=false 2024-11-12T14:53:20,175 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3945e1ca6f47,44367,1731423194517 expired; onlineServers=1 2024-11-12T14:53:20,175 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3945e1ca6f47,39415,1731423194147] 2024-11-12T14:53:20,176 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3945e1ca6f47,39415,1731423194147 already deleted, retry=false 2024-11-12T14:53:20,176 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3945e1ca6f47,39415,1731423194147 expired; onlineServers=0 2024-11-12T14:53:20,176 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3945e1ca6f47,40201,1731423193207' ***** 2024-11-12T14:53:20,176 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T14:53:20,176 INFO [M:0;3945e1ca6f47:40201 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:53:20,176 INFO [M:0;3945e1ca6f47:40201 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:53:20,177 DEBUG [M:0;3945e1ca6f47:40201 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T14:53:20,177 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T14:53:20,177 DEBUG [M:0;3945e1ca6f47:40201 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T14:53:20,177 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.large.0-1731423195975 {}] cleaner.HFileCleaner(306): Exit Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.large.0-1731423195975,5,FailOnTimeoutGroup] 2024-11-12T14:53:20,177 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.small.0-1731423195981 {}] cleaner.HFileCleaner(306): Exit Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.small.0-1731423195981,5,FailOnTimeoutGroup] 2024-11-12T14:53:20,177 INFO [M:0;3945e1ca6f47:40201 {}] hbase.ChoreService(370): Chore service for: master/3945e1ca6f47:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T14:53:20,177 INFO [M:0;3945e1ca6f47:40201 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:53:20,177 DEBUG [M:0;3945e1ca6f47:40201 {}] master.HMaster(1795): Stopping service threads 2024-11-12T14:53:20,177 INFO [M:0;3945e1ca6f47:40201 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T14:53:20,178 INFO [M:0;3945e1ca6f47:40201 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:53:20,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T14:53:20,178 INFO [M:0;3945e1ca6f47:40201 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:53:20,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,178 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T14:53:20,179 DEBUG [M:0;3945e1ca6f47:40201 {}] zookeeper.ZKUtil(347): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T14:53:20,179 WARN [M:0;3945e1ca6f47:40201 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T14:53:20,179 INFO [M:0;3945e1ca6f47:40201 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/.lastflushedseqids 2024-11-12T14:53:20,186 INFO [M:0;3945e1ca6f47:40201 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:53:20,187 INFO [M:0;3945e1ca6f47:40201 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T14:53:20,187 DEBUG [M:0;3945e1ca6f47:40201 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T14:53:20,187 INFO [M:0;3945e1ca6f47:40201 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:20,187 DEBUG [M:0;3945e1ca6f47:40201 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:20,187 DEBUG [M:0;3945e1ca6f47:40201 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T14:53:20,187 DEBUG [M:0;3945e1ca6f47:40201 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:20,187 INFO [M:0;3945e1ca6f47:40201 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-12T14:53:20,195 DEBUG [M:0;3945e1ca6f47:40201 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 70ac80c06b5e4a818251132439640983 2024-11-12T14:53:20,208 DEBUG [M:0;3945e1ca6f47:40201 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70ac80c06b5e4a818251132439640983 is 82, key is hbase:meta,,1/info:regioninfo/1731423197111/Put/seqid=0 2024-11-12T14:53:20,208 INFO [M:0;3945e1ca6f47:40201 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70ac80c06b5e4a818251132439640983 2024-11-12T14:53:20,215 DEBUG [M:0;3945e1ca6f47:40201 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dcb438f3f2d94166b610850c19fc6ed7 2024-11-12T14:53:20,228 DEBUG [M:0;3945e1ca6f47:40201 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dcb438f3f2d94166b610850c19fc6ed7 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731423197943/Put/seqid=0 2024-11-12T14:53:20,229 INFO [M:0;3945e1ca6f47:40201 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dcb438f3f2d94166b610850c19fc6ed7 2024-11-12T14:53:20,236 DEBUG [M:0;3945e1ca6f47:40201 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f45a6af15f504de09120e1f51d05572a 2024-11-12T14:53:20,248 DEBUG [M:0;3945e1ca6f47:40201 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f45a6af15f504de09120e1f51d05572a is 69, key is 3945e1ca6f47,39415,1731423194147/rs:state/1731423196049/Put/seqid=0 2024-11-12T14:53:20,249 INFO [M:0;3945e1ca6f47:40201 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f45a6af15f504de09120e1f51d05572a 2024-11-12T14:53:20,250 DEBUG [M:0;3945e1ca6f47:40201 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70ac80c06b5e4a818251132439640983 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/70ac80c06b5e4a818251132439640983 2024-11-12T14:53:20,252 INFO [M:0;3945e1ca6f47:40201 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/70ac80c06b5e4a818251132439640983, entries=8, sequenceid=84, filesize=5.5 K 2024-11-12T14:53:20,252 DEBUG [M:0;3945e1ca6f47:40201 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dcb438f3f2d94166b610850c19fc6ed7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dcb438f3f2d94166b610850c19fc6ed7 2024-11-12T14:53:20,254 INFO [M:0;3945e1ca6f47:40201 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dcb438f3f2d94166b610850c19fc6ed7, entries=9, sequenceid=84, filesize=6.5 K 2024-11-12T14:53:20,254 DEBUG [M:0;3945e1ca6f47:40201 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f45a6af15f504de09120e1f51d05572a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f45a6af15f504de09120e1f51d05572a 2024-11-12T14:53:20,256 INFO [M:0;3945e1ca6f47:40201 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f45a6af15f504de09120e1f51d05572a, entries=2, sequenceid=84, filesize=5.1 K 2024-11-12T14:53:20,256 INFO [M:0;3945e1ca6f47:40201 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 69ms, sequenceid=84, compaction requested=false 2024-11-12T14:53:20,258 INFO [M:0;3945e1ca6f47:40201 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:20,258 DEBUG [M:0;3945e1ca6f47:40201 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731423200187Disabling compacts and flushes for region at 1731423200187Disabling writes for close at 1731423200187Obtaining lock to block concurrent updates at 1731423200187Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731423200187Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731423200188 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731423200188Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731423200188Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731423200207 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731423200207Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731423200210 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731423200228 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731423200228Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731423200231 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731423200248 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731423200248Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d2ee87e: reopening flushed file at 1731423200250 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a80c17b: reopening flushed file at 1731423200252 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c24cb7d: reopening flushed file at 1731423200254 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 69ms, sequenceid=84, compaction requested=false at 1731423200256 (+2 ms)Writing region close event to WAL at 1731423200258 (+2 ms)Closed at 1731423200258 2024-11-12T14:53:20,260 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:53:20,260 INFO [M:0;3945e1ca6f47:40201 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T14:53:20,260 INFO [M:0;3945e1ca6f47:40201 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40201 2024-11-12T14:53:20,260 INFO [M:0;3945e1ca6f47:40201 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:53:20,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:20,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:20,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44367-0x100346ca3550002, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:20,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x100346ca3550001, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:20,274 INFO [RS:0;3945e1ca6f47:39415 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:53:20,274 INFO [RS:1;3945e1ca6f47:44367 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:53:20,275 INFO [RS:1;3945e1ca6f47:44367 {}] regionserver.HRegionServer(1031): Exiting; stopping=3945e1ca6f47,44367,1731423194517; zookeeper connection closed. 2024-11-12T14:53:20,275 INFO [RS:0;3945e1ca6f47:39415 {}] regionserver.HRegionServer(1031): Exiting; stopping=3945e1ca6f47,39415,1731423194147; zookeeper connection closed. 2024-11-12T14:53:20,275 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1b4499f7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1b4499f7 2024-11-12T14:53:20,275 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4c1381d2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4c1381d2 2024-11-12T14:53:20,276 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-12T14:53:20,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:20,362 INFO [M:0;3945e1ca6f47:40201 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:53:20,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40201-0x100346ca3550000, quorum=127.0.0.1:51303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:20,371 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T14:53:20,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-12T14:53:20,391 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:20,399 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: regionserver/3945e1ca6f47:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731423200040 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/3945e1ca6f47:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/3945e1ca6f47:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/3945e1ca6f47:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731423199888 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731423200037 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731423198339 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=359 (was 277) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7931 (was 8718) 2024-11-12T14:53:20,404 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=359, ProcessCount=11, AvailableMemoryMB=7931 2024-11-12T14:53:20,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/hadoop-log-dir so I do NOT create it in target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0 2024-11-12T14:53:20,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/hadoop-tmp-dir so I do NOT create it in target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0 2024-11-12T14:53:20,405 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:20,408 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/cluster_2c6fb005-c545-93c9-c02e-f30cbe1a0313, deleteOnExit=true 2024-11-12T14:53:20,410 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/cluster_2c6fb005-c545-93c9-c02e-f30cbe1a0313/zookeeper_0, clientPort=60193, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/cluster_2c6fb005-c545-93c9-c02e-f30cbe1a0313/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/cluster_2c6fb005-c545-93c9-c02e-f30cbe1a0313/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T14:53:20,411 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60193 2024-11-12T14:53:20,421 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23 with version=8 2024-11-12T14:53:20,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/hbase-staging 2024-11-12T14:53:20,423 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:53:20,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:20,424 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:20,424 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:53:20,424 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:20,424 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:53:20,424 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T14:53:20,424 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:53:20,425 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34965 2024-11-12T14:53:20,427 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34965 connecting to ZooKeeper ensemble=127.0.0.1:60193 2024-11-12T14:53:20,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:349650x0, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:53:20,431 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34965-0x100346cc1080000 connected 2024-11-12T14:53:20,445 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:20,446 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:20,447 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:53:20,447 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23, hbase.cluster.distributed=false 2024-11-12T14:53:20,450 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:53:20,450 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34965 2024-11-12T14:53:20,450 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34965 2024-11-12T14:53:20,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34965 2024-11-12T14:53:20,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34965 2024-11-12T14:53:20,452 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34965 2024-11-12T14:53:20,470 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:53:20,470 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:20,470 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:20,470 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:53:20,470 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:20,470 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:53:20,470 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T14:53:20,471 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:53:20,471 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46407 2024-11-12T14:53:20,473 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46407 connecting to ZooKeeper ensemble=127.0.0.1:60193 2024-11-12T14:53:20,473 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:20,473 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:20,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464070x0, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:53:20,476 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46407-0x100346cc1080001 connected 2024-11-12T14:53:20,476 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:53:20,477 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T14:53:20,477 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.cache 2024-11-12T14:53:20,477 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T14:53:20,478 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T14:53:20,478 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T14:53:20,478 INFO [Thread-118 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T14:53:20,478 WARN [Thread-118 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T14:53:20,478 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T14:53:20,479 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T14:53:20,479 INFO [Thread-118 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.persistence is complete. 2024-11-12T14:53:20,479 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T14:53:20,480 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T14:53:20,481 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:53:20,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46407 2024-11-12T14:53:20,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46407 2024-11-12T14:53:20,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46407 2024-11-12T14:53:20,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46407 2024-11-12T14:53:20,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46407 2024-11-12T14:53:20,499 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:53:20,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:20,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:20,499 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:53:20,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:20,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:53:20,499 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T14:53:20,500 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:53:20,500 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36205 2024-11-12T14:53:20,502 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36205 connecting to ZooKeeper ensemble=127.0.0.1:60193 2024-11-12T14:53:20,502 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:20,502 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:20,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:362050x0, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:53:20,505 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36205-0x100346cc1080002 connected 2024-11-12T14:53:20,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:53:20,505 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T14:53:20,506 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.cache 2024-11-12T14:53:20,506 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T14:53:20,506 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T14:53:20,506 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T14:53:20,507 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T14:53:20,507 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T14:53:20,507 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T14:53:20,507 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.persistence is complete. 2024-11-12T14:53:20,507 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T14:53:20,508 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T14:53:20,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T14:53:20,510 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:53:20,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36205 2024-11-12T14:53:20,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36205 2024-11-12T14:53:20,515 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36205 2024-11-12T14:53:20,519 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36205 2024-11-12T14:53:20,519 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36205 2024-11-12T14:53:20,531 DEBUG [M:0;3945e1ca6f47:34965 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3945e1ca6f47:34965 2024-11-12T14:53:20,531 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3945e1ca6f47,34965,1731423200423 2024-11-12T14:53:20,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:53:20,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:53:20,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:53:20,534 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3945e1ca6f47,34965,1731423200423 2024-11-12T14:53:20,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T14:53:20,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T14:53:20,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,537 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T14:53:20,537 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3945e1ca6f47,34965,1731423200423 from backup master directory 2024-11-12T14:53:20,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3945e1ca6f47,34965,1731423200423 2024-11-12T14:53:20,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:53:20,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:53:20,538 WARN [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:53:20,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:53:20,538 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3945e1ca6f47,34965,1731423200423 2024-11-12T14:53:20,540 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/hbase.id] with ID: d411606d-f3e0-4cc8-aab8-505a66236b51 2024-11-12T14:53:20,540 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/.tmp/hbase.id 2024-11-12T14:53:20,548 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/hbase.id] 2024-11-12T14:53:20,584 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:20,584 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T14:53:20,587 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-12T14:53:20,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,593 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:53:20,594 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T14:53:20,594 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:53:20,614 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store 2024-11-12T14:53:20,621 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:20,621 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T14:53:20,621 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:20,621 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:20,621 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T14:53:20,621 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:20,621 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:20,621 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731423200621Disabling compacts and flushes for region at 1731423200621Disabling writes for close at 1731423200621Writing region close event to WAL at 1731423200621Closed at 1731423200621 2024-11-12T14:53:20,622 WARN [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/.initializing 2024-11-12T14:53:20,622 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/WALs/3945e1ca6f47,34965,1731423200423 2024-11-12T14:53:20,627 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:53:20,631 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C34965%2C1731423200423, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/WALs/3945e1ca6f47,34965,1731423200423, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/oldWALs, maxLogs=10 2024-11-12T14:53:20,649 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/WALs/3945e1ca6f47,34965,1731423200423/3945e1ca6f47%2C34965%2C1731423200423.1731423200631 2024-11-12T14:53:20,650 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T14:53:20,650 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:53:20,650 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:20,651 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:20,651 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:20,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:20,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T14:53:20,659 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:20,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:20,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:20,665 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T14:53:20,665 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:20,665 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:53:20,665 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:20,669 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T14:53:20,669 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:20,669 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:53:20,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:20,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T14:53:20,673 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:20,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:53:20,674 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:20,675 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:20,675 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:20,675 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:20,675 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:20,675 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:53:20,676 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:53:20,681 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:53:20,682 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63019976, jitterRate=-0.060929179191589355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:53:20,683 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731423200651Initializing all the Stores at 1731423200651Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423200651Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423200652 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423200652Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423200652Cleaning up temporary data from old regions at 1731423200675 (+23 ms)Region opened successfully at 1731423200682 (+7 ms) 2024-11-12T14:53:20,683 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T14:53:20,687 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a6dcfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:53:20,688 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T14:53:20,688 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T14:53:20,688 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T14:53:20,688 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:53:20,689 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T14:53:20,689 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-12T14:53:20,689 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T14:53:20,691 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T14:53:20,692 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T14:53:20,693 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T14:53:20,693 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T14:53:20,694 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T14:53:20,695 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T14:53:20,695 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T14:53:20,696 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T14:53:20,697 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T14:53:20,698 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T14:53:20,699 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T14:53:20,701 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T14:53:20,702 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T14:53:20,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:53:20,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:53:20,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:53:20,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,705 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3945e1ca6f47,34965,1731423200423, sessionid=0x100346cc1080000, setting cluster-up flag (Was=false) 2024-11-12T14:53:20,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,710 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T14:53:20,711 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3945e1ca6f47,34965,1731423200423 2024-11-12T14:53:20,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,717 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T14:53:20,718 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3945e1ca6f47,34965,1731423200423 2024-11-12T14:53:20,720 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T14:53:20,721 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(746): ClusterId : d411606d-f3e0-4cc8-aab8-505a66236b51 2024-11-12T14:53:20,721 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(746): ClusterId : d411606d-f3e0-4cc8-aab8-505a66236b51 2024-11-12T14:53:20,721 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:53:20,721 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T14:53:20,721 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T14:53:20,722 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T14:53:20,722 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T14:53:20,722 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3945e1ca6f47,34965,1731423200423 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T14:53:20,723 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T14:53:20,723 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T14:53:20,723 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T14:53:20,723 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T14:53:20,724 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:53:20,724 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:53:20,724 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:53:20,724 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:53:20,724 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3945e1ca6f47:0, corePoolSize=10, maxPoolSize=10 2024-11-12T14:53:20,724 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,724 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:53:20,724 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,725 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T14:53:20,725 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T14:53:20,726 DEBUG [RS:0;3945e1ca6f47:46407 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eb3b318, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:53:20,726 DEBUG [RS:1;3945e1ca6f47:36205 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2321317a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:53:20,726 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:53:20,726 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T14:53:20,727 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:20,727 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T14:53:20,733 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731423230733 2024-11-12T14:53:20,734 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T14:53:20,734 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T14:53:20,734 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T14:53:20,734 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T14:53:20,734 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T14:53:20,734 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T14:53:20,734 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,735 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T14:53:20,736 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T14:53:20,736 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T14:53:20,736 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T14:53:20,736 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T14:53:20,737 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.large.0-1731423200736,5,FailOnTimeoutGroup] 2024-11-12T14:53:20,737 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.small.0-1731423200737,5,FailOnTimeoutGroup] 2024-11-12T14:53:20,737 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,738 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T14:53:20,738 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,738 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,746 DEBUG [RS:0;3945e1ca6f47:46407 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3945e1ca6f47:46407 2024-11-12T14:53:20,746 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T14:53:20,746 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T14:53:20,747 DEBUG [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T14:53:20,748 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(2659): reportForDuty to master=3945e1ca6f47,34965,1731423200423 with port=46407, startcode=1731423200469 2024-11-12T14:53:20,748 DEBUG [RS:0;3945e1ca6f47:46407 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T14:53:20,749 DEBUG [RS:1;3945e1ca6f47:36205 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;3945e1ca6f47:36205 2024-11-12T14:53:20,749 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T14:53:20,749 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T14:53:20,749 DEBUG [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T14:53:20,750 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(2659): reportForDuty to master=3945e1ca6f47,34965,1731423200423 with port=36205, startcode=1731423200499 2024-11-12T14:53:20,751 DEBUG [RS:1;3945e1ca6f47:36205 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T14:53:20,753 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T14:53:20,753 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23 2024-11-12T14:53:20,753 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43659, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T14:53:20,754 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34965 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:20,754 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34965 {}] master.ServerManager(517): Registering regionserver=3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:20,756 DEBUG [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23 2024-11-12T14:53:20,756 DEBUG [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T14:53:20,756 DEBUG [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T14:53:20,758 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56093, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T14:53:20,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:53:20,759 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34965 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:20,759 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34965 {}] master.ServerManager(517): Registering regionserver=3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:20,760 DEBUG [RS:0;3945e1ca6f47:46407 {}] zookeeper.ZKUtil(111): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:20,760 WARN [RS:0;3945e1ca6f47:46407 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:53:20,760 INFO [RS:0;3945e1ca6f47:46407 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:53:20,760 DEBUG [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:20,761 DEBUG [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23 2024-11-12T14:53:20,761 DEBUG [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T14:53:20,761 DEBUG [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T14:53:20,765 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3945e1ca6f47,46407,1731423200469] 2024-11-12T14:53:20,765 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3945e1ca6f47,36205,1731423200499] 2024-11-12T14:53:20,765 DEBUG [RS:1;3945e1ca6f47:36205 {}] zookeeper.ZKUtil(111): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:20,765 WARN [RS:1;3945e1ca6f47:36205 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:53:20,765 INFO [RS:1;3945e1ca6f47:36205 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:53:20,766 DEBUG [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:20,771 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T14:53:20,771 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T14:53:20,772 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:20,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T14:53:20,774 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T14:53:20,775 INFO [RS:0;3945e1ca6f47:46407 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:53:20,775 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,775 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T14:53:20,776 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T14:53:20,776 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,776 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T14:53:20,776 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,776 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,777 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,777 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,777 INFO [RS:1;3945e1ca6f47:36205 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:53:20,777 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,777 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,777 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:53:20,777 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,777 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T14:53:20,777 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,777 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,777 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,777 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,777 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,777 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T14:53:20,777 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:53:20,778 DEBUG [RS:0;3945e1ca6f47:46407 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:53:20,778 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,778 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,778 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,778 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,778 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,778 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,778 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:53:20,778 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,778 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,778 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,778 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,778 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,778 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:20,779 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:53:20,779 DEBUG [RS:1;3945e1ca6f47:36205 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:53:20,781 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,782 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,782 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,782 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,782 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,782 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,46407,1731423200469-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:53:20,785 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,785 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,785 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,786 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,786 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,786 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,36205,1731423200499-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:53:20,790 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T14:53:20,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:20,790 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:20,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T14:53:20,798 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T14:53:20,798 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:20,799 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:20,799 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T14:53:20,801 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T14:53:20,802 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,46407,1731423200469-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,802 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,802 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.Replication(171): 3945e1ca6f47,46407,1731423200469 started 2024-11-12T14:53:20,806 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T14:53:20,806 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T14:53:20,806 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:20,806 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,36205,1731423200499-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,806 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:20,806 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.Replication(171): 3945e1ca6f47,36205,1731423200499 started 2024-11-12T14:53:20,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T14:53:20,811 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T14:53:20,812 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:20,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:20,812 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T14:53:20,813 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740 2024-11-12T14:53:20,813 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740 2024-11-12T14:53:20,814 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T14:53:20,814 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T14:53:20,814 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:53:20,815 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T14:53:20,819 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,819 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(1482): Serving as 3945e1ca6f47,46407,1731423200469, RpcServer on 3945e1ca6f47/172.17.0.2:46407, sessionid=0x100346cc1080001 2024-11-12T14:53:20,819 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T14:53:20,819 DEBUG [RS:0;3945e1ca6f47:46407 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:20,819 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,46407,1731423200469' 2024-11-12T14:53:20,819 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T14:53:20,821 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T14:53:20,821 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:53:20,821 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74777725, jitterRate=0.11427493393421173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:53:20,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731423200772Initializing all the Stores at 1731423200772Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423200772Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423200772Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423200772Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423200772Cleaning up temporary data from old regions at 1731423200814 (+42 ms)Region opened successfully at 1731423200822 (+8 ms) 2024-11-12T14:53:20,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T14:53:20,822 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T14:53:20,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T14:53:20,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T14:53:20,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T14:53:20,823 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T14:53:20,823 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T14:53:20,823 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T14:53:20,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731423200822Disabling compacts and flushes for region at 1731423200822Disabling writes for close at 1731423200822Writing region close event to WAL at 1731423200823 (+1 ms)Closed at 1731423200823 2024-11-12T14:53:20,823 DEBUG [RS:0;3945e1ca6f47:46407 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:20,823 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,46407,1731423200469' 2024-11-12T14:53:20,823 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T14:53:20,824 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T14:53:20,824 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:53:20,824 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T14:53:20,824 DEBUG [RS:0;3945e1ca6f47:46407 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T14:53:20,824 INFO [RS:0;3945e1ca6f47:46407 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T14:53:20,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T14:53:20,824 INFO [RS:0;3945e1ca6f47:46407 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T14:53:20,824 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:20,825 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(1482): Serving as 3945e1ca6f47,36205,1731423200499, RpcServer on 3945e1ca6f47/172.17.0.2:36205, sessionid=0x100346cc1080002 2024-11-12T14:53:20,825 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T14:53:20,825 DEBUG [RS:1;3945e1ca6f47:36205 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:20,825 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,36205,1731423200499' 2024-11-12T14:53:20,825 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T14:53:20,825 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:53:20,826 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T14:53:20,826 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T14:53:20,826 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T14:53:20,826 DEBUG [RS:1;3945e1ca6f47:36205 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:20,826 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,36205,1731423200499' 2024-11-12T14:53:20,826 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T14:53:20,827 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T14:53:20,827 DEBUG [RS:1;3945e1ca6f47:36205 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T14:53:20,827 INFO [RS:1;3945e1ca6f47:36205 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T14:53:20,827 INFO [RS:1;3945e1ca6f47:36205 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T14:53:20,828 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T14:53:20,925 INFO [RS:0;3945e1ca6f47:46407 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:53:20,927 INFO [RS:0;3945e1ca6f47:46407 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3945e1ca6f47%2C46407%2C1731423200469, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,46407,1731423200469, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs, maxLogs=55 2024-11-12T14:53:20,928 INFO [RS:1;3945e1ca6f47:36205 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:53:20,930 INFO [RS:1;3945e1ca6f47:36205 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3945e1ca6f47%2C36205%2C1731423200499, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,36205,1731423200499, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs, maxLogs=55 2024-11-12T14:53:20,950 INFO [RS:0;3945e1ca6f47:46407 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,46407,1731423200469/3945e1ca6f47%2C46407%2C1731423200469.1731423200931 2024-11-12T14:53:20,951 DEBUG [RS:0;3945e1ca6f47:46407 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T14:53:20,951 INFO [RS:1;3945e1ca6f47:36205 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,36205,1731423200499/3945e1ca6f47%2C36205%2C1731423200499.1731423200931 2024-11-12T14:53:20,951 DEBUG [RS:1;3945e1ca6f47:36205 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T14:53:20,978 DEBUG [3945e1ca6f47:34965 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-12T14:53:20,978 DEBUG [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(204): Hosts are {3945e1ca6f47=0} racks are {/default-rack=0} 2024-11-12T14:53:20,979 DEBUG [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T14:53:20,979 DEBUG [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T14:53:20,979 DEBUG [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T14:53:20,979 DEBUG [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T14:53:20,980 INFO [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T14:53:20,980 INFO [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T14:53:20,980 DEBUG [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T14:53:20,980 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:20,982 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,36205,1731423200499, state=OPENING 2024-11-12T14:53:20,983 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T14:53:20,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:20,987 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:20,987 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:20,987 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:20,987 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:53:20,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,36205,1731423200499}] 2024-11-12T14:53:21,142 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:53:21,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34067, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:53:21,147 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T14:53:21,147 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:53:21,147 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T14:53:21,148 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3945e1ca6f47%2C36205%2C1731423200499.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,36205,1731423200499, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs, maxLogs=55 2024-11-12T14:53:21,165 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,36205,1731423200499/3945e1ca6f47%2C36205%2C1731423200499.meta.1731423201149.meta 2024-11-12T14:53:21,166 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T14:53:21,166 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:53:21,167 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T14:53:21,167 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T14:53:21,167 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T14:53:21,167 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T14:53:21,167 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:21,167 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T14:53:21,167 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T14:53:21,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T14:53:21,169 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T14:53:21,169 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:21,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:21,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T14:53:21,170 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T14:53:21,170 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:21,170 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:21,170 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T14:53:21,171 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T14:53:21,171 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:21,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:21,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T14:53:21,172 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T14:53:21,172 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:21,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:21,172 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T14:53:21,173 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740 2024-11-12T14:53:21,173 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740 2024-11-12T14:53:21,173 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T14:53:21,173 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T14:53:21,173 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:53:21,174 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T14:53:21,174 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62051656, jitterRate=-0.07535827159881592}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:53:21,174 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T14:53:21,175 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731423201168Writing region info on filesystem at 1731423201168Initializing all the Stores at 1731423201168Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423201168Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423201168Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423201168Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423201168Cleaning up temporary data from old regions at 1731423201173 (+5 ms)Running coprocessor post-open hooks at 1731423201174 (+1 ms)Region opened successfully at 1731423201175 (+1 ms) 2024-11-12T14:53:21,176 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731423201141 2024-11-12T14:53:21,178 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T14:53:21,179 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T14:53:21,179 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:21,179 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,36205,1731423200499, state=OPEN 2024-11-12T14:53:21,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:53:21,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:53:21,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:53:21,181 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:21,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:21,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:21,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:21,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T14:53:21,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,36205,1731423200499 in 194 msec 2024-11-12T14:53:21,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T14:53:21,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 359 msec 2024-11-12T14:53:21,186 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:53:21,186 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T14:53:21,189 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:53:21,190 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,36205,1731423200499, seqNum=-1] 2024-11-12T14:53:21,190 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:53:21,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59231, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:53:21,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 475 msec 2024-11-12T14:53:21,197 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731423201197, completionTime=-1 2024-11-12T14:53:21,197 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-12T14:53:21,197 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:53:21,199 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T14:53:21,199 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731423261199 2024-11-12T14:53:21,199 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731423321199 2024-11-12T14:53:21,199 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-12T14:53:21,200 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,34965,1731423200423-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:21,200 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,34965,1731423200423-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:21,200 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,34965,1731423200423-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:21,200 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3945e1ca6f47:34965, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:21,200 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:21,200 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:21,203 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T14:53:21,205 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.666sec 2024-11-12T14:53:21,205 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T14:53:21,205 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T14:53:21,205 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T14:53:21,205 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T14:53:21,205 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T14:53:21,205 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,34965,1731423200423-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:53:21,205 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,34965,1731423200423-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T14:53:21,206 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T14:53:21,206 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T14:53:21,207 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,34965,1731423200423-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:21,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f5b2b42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:53:21,222 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3945e1ca6f47,34965,-1 for getting cluster id 2024-11-12T14:53:21,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:53:21,224 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd411606d-f3e0-4cc8-aab8-505a66236b51' 2024-11-12T14:53:21,224 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:53:21,224 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d411606d-f3e0-4cc8-aab8-505a66236b51" 2024-11-12T14:53:21,225 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dca3a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:53:21,225 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3945e1ca6f47,34965,-1] 2024-11-12T14:53:21,225 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:53:21,225 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:21,227 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:53:21,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ab0064, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:53:21,230 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:53:21,232 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,36205,1731423200499, seqNum=-1] 2024-11-12T14:53:21,233 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:53:21,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:53:21,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3945e1ca6f47,34965,1731423200423 2024-11-12T14:53:21,238 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T14:53:21,239 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3945e1ca6f47,34965,1731423200423 2024-11-12T14:53:21,239 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@8d8de05 2024-11-12T14:53:21,240 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T14:53:21,242 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T14:53:21,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34965 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731423201238', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:53:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34965 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731423201238 2024-11-12T14:53:21,245 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731423201238 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T14:53:21,245 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:21,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34965 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731423201238" procId is: 4 2024-11-12T14:53:21,246 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731423201238 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T14:53:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34965 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:53:21,257 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731423201238-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 573725d9ded94d93e3ea87860c9c7bca, NAME => 'testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731423201238', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23 2024-11-12T14:53:21,264 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731423201238-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:21,264 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731423201238-pool-0 {}] regionserver.HRegion(1722): Closing 573725d9ded94d93e3ea87860c9c7bca, disabling compactions & flushes 2024-11-12T14:53:21,264 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731423201238-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:21,264 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731423201238-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:21,264 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731423201238-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. after waiting 0 ms 2024-11-12T14:53:21,264 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731423201238-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:21,264 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731423201238-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:21,264 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731423201238-pool-0 {}] regionserver.HRegion(1676): Region close journal for 573725d9ded94d93e3ea87860c9c7bca: Waiting for close lock at 1731423201264Disabling compacts and flushes for region at 1731423201264Disabling writes for close at 1731423201264Writing region close event to WAL at 1731423201264Closed at 1731423201264 2024-11-12T14:53:21,266 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731423201238 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T14:53:21,266 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731423201266"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731423201266"}]},"ts":"1731423201266"} 2024-11-12T14:53:21,269 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T14:53:21,271 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731423201238 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T14:53:21,271 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731423201238","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731423201271"}]},"ts":"1731423201271"} 2024-11-12T14:53:21,274 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731423201238, state=ENABLING in hbase:meta 2024-11-12T14:53:21,274 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {3945e1ca6f47=0} racks are {/default-rack=0} 2024-11-12T14:53:21,275 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T14:53:21,275 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T14:53:21,275 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T14:53:21,275 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T14:53:21,275 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T14:53:21,275 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T14:53:21,275 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T14:53:21,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731423201238, region=573725d9ded94d93e3ea87860c9c7bca, ASSIGN}] 2024-11-12T14:53:21,277 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731423201238, region=573725d9ded94d93e3ea87860c9c7bca, ASSIGN 2024-11-12T14:53:21,278 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731423201238, region=573725d9ded94d93e3ea87860c9c7bca, ASSIGN; state=OFFLINE, location=3945e1ca6f47,36205,1731423200499; forceNewPlan=false, retain=false 2024-11-12T14:53:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34965 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:53:21,428 INFO [3945e1ca6f47:34965 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T14:53:21,429 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=573725d9ded94d93e3ea87860c9c7bca, regionState=OPENING, regionLocation=3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:21,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731423201238, region=573725d9ded94d93e3ea87860c9c7bca, ASSIGN because future has completed 2024-11-12T14:53:21,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 573725d9ded94d93e3ea87860c9c7bca, server=3945e1ca6f47,36205,1731423200499}] 2024-11-12T14:53:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34965 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:53:21,591 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:21,591 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 573725d9ded94d93e3ea87860c9c7bca, NAME => 'testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:53:21,591 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731423201238 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:21,591 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:21,591 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:21,592 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:21,592 INFO [StoreOpener-573725d9ded94d93e3ea87860c9c7bca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:21,595 INFO [StoreOpener-573725d9ded94d93e3ea87860c9c7bca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 573725d9ded94d93e3ea87860c9c7bca columnFamilyName family 2024-11-12T14:53:21,595 DEBUG [StoreOpener-573725d9ded94d93e3ea87860c9c7bca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:21,596 INFO [StoreOpener-573725d9ded94d93e3ea87860c9c7bca-1 {}] regionserver.HStore(327): Store=573725d9ded94d93e3ea87860c9c7bca/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:53:21,596 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:21,596 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:21,597 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:21,597 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:21,597 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:21,597 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:21,603 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:53:21,603 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 573725d9ded94d93e3ea87860c9c7bca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61792271, jitterRate=-0.07922340929508209}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:53:21,603 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:21,604 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 573725d9ded94d93e3ea87860c9c7bca: Running coprocessor pre-open hook at 1731423201592Writing region info on filesystem at 1731423201592Initializing all the Stores at 1731423201592Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423201592Cleaning up temporary data from old regions at 1731423201597 (+5 ms)Running coprocessor post-open hooks at 1731423201603 (+6 ms)Region opened successfully at 1731423201604 (+1 ms) 2024-11-12T14:53:21,605 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca., pid=6, masterSystemTime=1731423201588 2024-11-12T14:53:21,607 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:21,607 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:21,608 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=573725d9ded94d93e3ea87860c9c7bca, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:21,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 573725d9ded94d93e3ea87860c9c7bca, server=3945e1ca6f47,36205,1731423200499 because future has completed 2024-11-12T14:53:21,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T14:53:21,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 573725d9ded94d93e3ea87860c9c7bca, server=3945e1ca6f47,36205,1731423200499 in 178 msec 2024-11-12T14:53:21,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T14:53:21,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731423201238, region=573725d9ded94d93e3ea87860c9c7bca, ASSIGN in 339 msec 2024-11-12T14:53:21,616 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731423201238 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T14:53:21,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731423201238","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731423201616"}]},"ts":"1731423201616"} 2024-11-12T14:53:21,619 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731423201238, state=ENABLED in hbase:meta 2024-11-12T14:53:21,620 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731423201238 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T14:53:21,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731423201238 in 377 msec 2024-11-12T14:53:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34965 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:53:21,876 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731423201238 completed 2024-11-12T14:53:21,876 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731423201238 get assigned. Timeout = 60000ms 2024-11-12T14:53:21,877 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T14:53:21,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731423201238 assigned to meta. Checking AM states. 2024-11-12T14:53:21,885 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T14:53:21,885 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731423201238 assigned. 2024-11-12T14:53:21,889 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731423201238', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca., hostname=3945e1ca6f47,36205,1731423200499, seqNum=2] 2024-11-12T14:53:21,895 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 573725d9ded94d93e3ea87860c9c7bca 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-12T14:53:21,903 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b6222671e8864a4691b329bb215a5f30 2024-11-12T14:53:21,917 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/.tmp/family/b6222671e8864a4691b329bb215a5f30 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-12T14:53:21,917 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/.tmp/family/b6222671e8864a4691b329bb215a5f30 2024-11-12T14:53:21,920 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/.tmp/family/b6222671e8864a4691b329bb215a5f30 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/family/b6222671e8864a4691b329bb215a5f30 2024-11-12T14:53:21,921 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/family/b6222671e8864a4691b329bb215a5f30, delay=963 ms 2024-11-12T14:53:21,922 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/family/b6222671e8864a4691b329bb215a5f30, entries=2, sequenceid=6, filesize=4.9 K 2024-11-12T14:53:21,923 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 573725d9ded94d93e3ea87860c9c7bca in 27ms, sequenceid=6, compaction requested=false 2024-11-12T14:53:21,923 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731423201238' 2024-11-12T14:53:21,923 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 573725d9ded94d93e3ea87860c9c7bca: 2024-11-12T14:53:22,573 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:53:22,886 DEBUG [hfile-prefetch-1731423199888 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b6222671e8864a4691b329bb215a5f30_0. 2024-11-12T14:53:22,887 DEBUG [hfile-prefetch-1731423199888 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b6222671e8864a4691b329bb215a5f30_117. 2024-11-12T14:53:22,888 DEBUG [hfile-prefetch-1731423199888 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/family/b6222671e8864a4691b329bb215a5f30, with total blocks 2, and data blocks 1 2024-11-12T14:53:22,888 DEBUG [hfile-prefetch-1731423199888 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-12T14:53:22,888 DEBUG [hfile-prefetch-1731423199888 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/family/b6222671e8864a4691b329bb215a5f30, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-12T14:53:22,888 DEBUG [hfile-prefetch-1731423199888 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/family/b6222671e8864a4691b329bb215a5f30. Will sleep 100ms and try the verification again. 2024-11-12T14:53:22,909 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-12T14:53:22,909 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-12T14:53:22,910 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T14:53:22,910 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T14:53:22,924 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;3945e1ca6f47:36205,5,FailOnTimeoutGroup] 2024-11-12T14:53:22,924 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3945e1ca6f47,36205,1731423200499' ***** 2024-11-12T14:53:22,924 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-12T14:53:22,925 INFO [RS:1;3945e1ca6f47:36205 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T14:53:22,925 INFO [RS:1;3945e1ca6f47:36205 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.persistence 2024-11-12T14:53:22,925 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T14:53:22,925 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T14:53:22,925 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T14:53:22,925 INFO [RS:1;3945e1ca6f47:36205 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T14:53:22,925 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-12T14:53:22,925 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731423202910 2024-11-12T14:53:22,926 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-12T14:53:22,926 INFO [bucket-cache-persister {}] bucket.BucketCachePersister(62): Finishing cache persister thread. 2024-11-12T14:53:22,935 DEBUG [RS:1;3945e1ca6f47:36205 {}] bucket.BucketCache(1488): Thread RS:1;3945e1ca6f47:36205 started persisting bucket cache to file 2024-11-12T14:53:22,935 DEBUG [RS:1;3945e1ca6f47:36205 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T14:53:22,935 DEBUG [RS:1;3945e1ca6f47:36205 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T14:53:22,938 DEBUG [RS:1;3945e1ca6f47:36205 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-12T14:53:22,938 DEBUG [RS:1;3945e1ca6f47:36205 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731423202935 2024-11-12T14:53:22,938 DEBUG [RS:1;3945e1ca6f47:36205 {}] bucket.BucketCache(1510): Thread RS:1;3945e1ca6f47:36205 finished persisting bucket cache to file, renaming 2024-11-12T14:53:22,939 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T14:53:22,939 INFO [RS:1;3945e1ca6f47:36205 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T14:53:22,939 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T14:53:22,939 INFO [RS:1;3945e1ca6f47:36205 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T14:53:22,939 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(3091): Received CLOSE for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:22,939 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(959): stopping server 3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:22,939 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:53:22,939 INFO [RS:1;3945e1ca6f47:36205 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;3945e1ca6f47:36205. 2024-11-12T14:53:22,940 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T14:53:22,940 DEBUG [RS:1;3945e1ca6f47:36205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:53:22,940 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 573725d9ded94d93e3ea87860c9c7bca, disabling compactions & flushes 2024-11-12T14:53:22,940 DEBUG [RS:1;3945e1ca6f47:36205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:22,940 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:22,940 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:22,940 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T14:53:22,940 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. after waiting 0 ms 2024-11-12T14:53:22,940 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T14:53:22,940 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:22,940 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T14:53:22,940 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T14:53:22,941 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-12T14:53:22,941 DEBUG [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 573725d9ded94d93e3ea87860c9c7bca=testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca.} 2024-11-12T14:53:22,941 DEBUG [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:22,941 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T14:53:22,941 DEBUG [StoreFileCloser-573725d9ded94d93e3ea87860c9c7bca-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/family/b6222671e8864a4691b329bb215a5f30 2024-11-12T14:53:22,941 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T14:53:22,941 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T14:53:22,941 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T14:53:22,941 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T14:53:22,941 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T14:53:22,941 WARN [hfile-prefetch-1731423199888 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/family/b6222671e8864a4691b329bb215a5f30, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-12T14:53:22,942 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-12T14:53:22,942 DEBUG [hfile-prefetch-1731423199888 {}] hfile.PrefetchExecutor(121): Prefetch completed for b6222671e8864a4691b329bb215a5f30 2024-11-12T14:53:22,947 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-12T14:53:22,948 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:22,948 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 573725d9ded94d93e3ea87860c9c7bca: Waiting for close lock at 1731423202939Running coprocessor pre-close hooks at 1731423202940 (+1 ms)Disabling compacts and flushes for region at 1731423202940Disabling writes for close at 1731423202940Writing region close event to WAL at 1731423202941 (+1 ms)Running coprocessor post-close hooks at 1731423202948 (+7 ms)Closed at 1731423202948 2024-11-12T14:53:22,948 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:22,954 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 88c67ecafde047de8696dc3921320d8c 2024-11-12T14:53:22,967 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/info/88c67ecafde047de8696dc3921320d8c is 209, key is testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca./info:regioninfo/1731423201608/Put/seqid=0 2024-11-12T14:53:22,968 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/info/88c67ecafde047de8696dc3921320d8c 2024-11-12T14:53:22,975 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f738e7dadb7448d9a5be4074e6899528 2024-11-12T14:53:22,988 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/ns/f738e7dadb7448d9a5be4074e6899528 is 43, key is default/ns:d/1731423201192/Put/seqid=0 2024-11-12T14:53:22,988 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/ns/f738e7dadb7448d9a5be4074e6899528 2024-11-12T14:53:22,991 INFO [regionserver/3945e1ca6f47:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:53:22,996 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 881feece945349a1b0248a0b9de66fe8 2024-11-12T14:53:23,008 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/table/881feece945349a1b0248a0b9de66fe8 is 80, key is testBlockEvictionOnGracefulStop1731423201238/table:state/1731423201616/Put/seqid=0 2024-11-12T14:53:23,009 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/table/881feece945349a1b0248a0b9de66fe8 2024-11-12T14:53:23,011 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/info/88c67ecafde047de8696dc3921320d8c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/88c67ecafde047de8696dc3921320d8c 2024-11-12T14:53:23,013 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/88c67ecafde047de8696dc3921320d8c, delay=933 ms 2024-11-12T14:53:23,014 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/88c67ecafde047de8696dc3921320d8c, entries=10, sequenceid=11, filesize=7.0 K 2024-11-12T14:53:23,014 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/ns/f738e7dadb7448d9a5be4074e6899528 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/ns/f738e7dadb7448d9a5be4074e6899528 2024-11-12T14:53:23,015 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/ns/f738e7dadb7448d9a5be4074e6899528, delay=976 ms 2024-11-12T14:53:23,016 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/ns/f738e7dadb7448d9a5be4074e6899528, entries=2, sequenceid=11, filesize=5.0 K 2024-11-12T14:53:23,016 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/table/881feece945349a1b0248a0b9de66fe8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/table/881feece945349a1b0248a0b9de66fe8 2024-11-12T14:53:23,017 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/table/881feece945349a1b0248a0b9de66fe8, delay=923 ms 2024-11-12T14:53:23,018 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/table/881feece945349a1b0248a0b9de66fe8, entries=2, sequenceid=11, filesize=5.3 K 2024-11-12T14:53:23,019 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=11, compaction requested=false 2024-11-12T14:53:23,020 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/88c67ecafde047de8696dc3921320d8c 2024-11-12T14:53:23,020 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/ns/f738e7dadb7448d9a5be4074e6899528 2024-11-12T14:53:23,021 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/table/881feece945349a1b0248a0b9de66fe8 2024-11-12T14:53:23,025 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-12T14:53:23,025 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T14:53:23,026 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T14:53:23,026 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731423202941Running coprocessor pre-close hooks at 1731423202941Disabling compacts and flushes for region at 1731423202941Disabling writes for close at 1731423202941Obtaining lock to block concurrent updates at 1731423202942 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731423202942Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731423202942Flushing stores of hbase:meta,,1.1588230740 at 1731423202943 (+1 ms)Flushing 1588230740/info: creating writer at 1731423202943Flushing 1588230740/info: appending metadata at 1731423202967 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731423202967Flushing 1588230740/ns: creating writer at 1731423202970 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731423202987 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731423202987Flushing 1588230740/table: creating writer at 1731423202991 (+4 ms)Flushing 1588230740/table: appending metadata at 1731423203008 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731423203008Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ea29a73: reopening flushed file at 1731423203011 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@143b660f: reopening flushed file at 1731423203014 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@542e8aa1: reopening flushed file at 1731423203016 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=11, compaction requested=false at 1731423203019 (+3 ms)Writing region close event to WAL at 1731423203021 (+2 ms)Running coprocessor post-close hooks at 1731423203025 (+4 ms)Closed at 1731423203026 (+1 ms) 2024-11-12T14:53:23,026 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T14:53:23,141 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(976): stopping server 3945e1ca6f47,36205,1731423200499; all regions closed. 2024-11-12T14:53:23,144 DEBUG [RS:1;3945e1ca6f47:36205 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs 2024-11-12T14:53:23,144 INFO [RS:1;3945e1ca6f47:36205 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C36205%2C1731423200499.meta:.meta(num 1731423201149) 2024-11-12T14:53:23,145 DEBUG [RS:1;3945e1ca6f47:36205 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs 2024-11-12T14:53:23,145 INFO [RS:1;3945e1ca6f47:36205 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C36205%2C1731423200499:(num 1731423200931) 2024-11-12T14:53:23,145 DEBUG [RS:1;3945e1ca6f47:36205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:23,145 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:53:23,146 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:53:23,146 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.ChoreService(370): Chore service for: regionserver/3945e1ca6f47:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T14:53:23,146 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:53:23,146 INFO [regionserver/3945e1ca6f47:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:53:23,146 INFO [RS:1;3945e1ca6f47:36205 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36205 2024-11-12T14:53:23,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:53:23,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3945e1ca6f47,36205,1731423200499 2024-11-12T14:53:23,150 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:53:23,151 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3945e1ca6f47,36205,1731423200499] 2024-11-12T14:53:23,153 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3945e1ca6f47,36205,1731423200499 already deleted, retry=false 2024-11-12T14:53:23,153 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 3945e1ca6f47,36205,1731423200499 on 3945e1ca6f47,34965,1731423200423 2024-11-12T14:53:23,157 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3945e1ca6f47,36205,1731423200499, splitWal=true, meta=true 2024-11-12T14:53:23,160 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 3945e1ca6f47,36205,1731423200499 (carryingMeta=true) 3945e1ca6f47,36205,1731423200499/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3e3eae54[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-12T14:53:23,160 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 3945e1ca6f47,36205,1731423200499, splitWal=true, meta=true 2024-11-12T14:53:23,161 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,36205,1731423200499, splitWal=true, meta=true, isMeta: true 2024-11-12T14:53:23,161 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,36205,1731423200499-splitting 2024-11-12T14:53:23,161 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,36205,1731423200499-splitting dir is empty, no logs to split. 2024-11-12T14:53:23,161 INFO [PEWorker-2 {}] master.SplitWALManager(105): 3945e1ca6f47,36205,1731423200499 WAL count=0, meta=true 2024-11-12T14:53:23,163 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,36205,1731423200499-splitting dir is empty, no logs to split. 2024-11-12T14:53:23,163 INFO [PEWorker-2 {}] master.SplitWALManager(105): 3945e1ca6f47,36205,1731423200499 WAL count=0, meta=true 2024-11-12T14:53:23,163 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 3945e1ca6f47,36205,1731423200499 WAL splitting is done? wals=0, meta=true 2024-11-12T14:53:23,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T14:53:23,165 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:53:23,166 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:53:23,252 INFO [RS:1;3945e1ca6f47:36205 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:53:23,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:23,252 INFO [RS:1;3945e1ca6f47:36205 {}] regionserver.HRegionServer(1031): Exiting; stopping=3945e1ca6f47,36205,1731423200499; zookeeper connection closed. 2024-11-12T14:53:23,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36205-0x100346cc1080002, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:23,252 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6dfcbe09 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6dfcbe09 2024-11-12T14:53:23,316 DEBUG [3945e1ca6f47:34965 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T14:53:23,317 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:23,319 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,46407,1731423200469, state=OPENING 2024-11-12T14:53:23,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:53:23,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:53:23,322 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:53:23,323 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:23,323 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:23,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,46407,1731423200469}] 2024-11-12T14:53:23,445 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:53:23,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:23,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:23,445 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:53:23,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:53:23,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:53:23,445 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T14:53:23,445 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:53:23,446 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45215 2024-11-12T14:53:23,447 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45215 connecting to ZooKeeper ensemble=127.0.0.1:60193 2024-11-12T14:53:23,447 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:23,448 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:23,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452150x0, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:53:23,450 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45215-0x100346cc1080003 connected 2024-11-12T14:53:23,450 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45215-0x100346cc1080003, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-12T14:53:23,450 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-12T14:53:23,451 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T14:53:23,453 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.cache 2024-11-12T14:53:23,453 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T14:53:23,453 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T14:53:23,454 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T14:53:23,454 INFO [Thread-206 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T14:53:23,454 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T14:53:23,454 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T14:53:23,455 INFO [Thread-206 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-12T14:53:23,455 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45215-0x100346cc1080003, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T14:53:23,456 INFO [Thread-206 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-12T14:53:23,457 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45215-0x100346cc1080003, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:53:23,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45215 2024-11-12T14:53:23,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45215 2024-11-12T14:53:23,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45215 2024-11-12T14:53:23,461 DEBUG [Thread-206 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-12T14:53:23,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45215 2024-11-12T14:53:23,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45215 2024-11-12T14:53:23,462 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T14:53:23,462 INFO [Thread-206 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-12T14:53:23,462 INFO [Thread-206 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.persistence is complete. 2024-11-12T14:53:23,463 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.HRegionServer(746): ClusterId : d411606d-f3e0-4cc8-aab8-505a66236b51 2024-11-12T14:53:23,463 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T14:53:23,465 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T14:53:23,465 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T14:53:23,467 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T14:53:23,468 DEBUG [RS:2;3945e1ca6f47:45215 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8bb6ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:53:23,477 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:53:23,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43957, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:53:23,479 DEBUG [RS:2;3945e1ca6f47:45215 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;3945e1ca6f47:45215 2024-11-12T14:53:23,479 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T14:53:23,479 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T14:53:23,479 DEBUG [RS:2;3945e1ca6f47:45215 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T14:53:23,480 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.HRegionServer(2659): reportForDuty to master=3945e1ca6f47,34965,1731423200423 with port=45215, startcode=1731423203444 2024-11-12T14:53:23,480 DEBUG [RS:2;3945e1ca6f47:45215 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T14:53:23,480 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T14:53:23,480 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:53:23,481 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T14:53:23,481 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3945e1ca6f47%2C46407%2C1731423200469.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,46407,1731423200469, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs, maxLogs=55 2024-11-12T14:53:23,482 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57965, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T14:53:23,483 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34965 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3945e1ca6f47,45215,1731423203444 2024-11-12T14:53:23,483 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34965 {}] master.ServerManager(517): Registering regionserver=3945e1ca6f47,45215,1731423203444 2024-11-12T14:53:23,484 DEBUG [RS:2;3945e1ca6f47:45215 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23 2024-11-12T14:53:23,484 DEBUG [RS:2;3945e1ca6f47:45215 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T14:53:23,484 DEBUG [RS:2;3945e1ca6f47:45215 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T14:53:23,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:53:23,486 DEBUG [RS:2;3945e1ca6f47:45215 {}] zookeeper.ZKUtil(111): regionserver:45215-0x100346cc1080003, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3945e1ca6f47,45215,1731423203444 2024-11-12T14:53:23,486 WARN [RS:2;3945e1ca6f47:45215 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:53:23,486 INFO [RS:2;3945e1ca6f47:45215 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:53:23,487 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3945e1ca6f47,45215,1731423203444] 2024-11-12T14:53:23,487 DEBUG [RS:2;3945e1ca6f47:45215 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,45215,1731423203444 2024-11-12T14:53:23,491 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T14:53:23,493 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T14:53:23,494 INFO [RS:2;3945e1ca6f47:45215 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:53:23,494 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:23,494 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T14:53:23,494 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T14:53:23,495 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:23,495 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:23,495 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:23,495 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:23,495 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:23,495 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:23,495 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:53:23,495 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:23,495 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:23,495 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:23,495 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:23,495 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:23,496 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:53:23,496 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:53:23,496 DEBUG [RS:2;3945e1ca6f47:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:53:23,496 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:23,496 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:23,496 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:23,496 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:23,496 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:23,496 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,45215,1731423203444-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:53:23,500 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,46407,1731423200469/3945e1ca6f47%2C46407%2C1731423200469.meta.1731423203482.meta 2024-11-12T14:53:23,500 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T14:53:23,500 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:53:23,500 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T14:53:23,501 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T14:53:23,501 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T14:53:23,501 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T14:53:23,501 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:23,501 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T14:53:23,501 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T14:53:23,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T14:53:23,502 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T14:53:23,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:23,504 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/88c67ecafde047de8696dc3921320d8c, delay=976 ms 2024-11-12T14:53:23,505 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/88c67ecafde047de8696dc3921320d8c 2024-11-12T14:53:23,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:23,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T14:53:23,506 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T14:53:23,506 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:23,508 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/ns/f738e7dadb7448d9a5be4074e6899528, delay=923 ms 2024-11-12T14:53:23,509 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/ns/f738e7dadb7448d9a5be4074e6899528 2024-11-12T14:53:23,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:23,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T14:53:23,510 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T14:53:23,510 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:23,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:23,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T14:53:23,510 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T14:53:23,511 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:23,511 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T14:53:23,511 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,45215,1731423203444-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:23,511 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:23,512 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.Replication(171): 3945e1ca6f47,45215,1731423203444 started 2024-11-12T14:53:23,512 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/table/881feece945349a1b0248a0b9de66fe8, delay=991 ms 2024-11-12T14:53:23,513 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/table/881feece945349a1b0248a0b9de66fe8 2024-11-12T14:53:23,514 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:53:23,514 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T14:53:23,514 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740 2024-11-12T14:53:23,515 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740 2024-11-12T14:53:23,515 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T14:53:23,515 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T14:53:23,516 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:53:23,517 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T14:53:23,517 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69430485, jitterRate=0.03459484875202179}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:53:23,517 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T14:53:23,518 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731423203501Writing region info on filesystem at 1731423203501Initializing all the Stores at 1731423203501Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423203501Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423203502 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423203502Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423203502Cleaning up temporary data from old regions at 1731423203515 (+13 ms)Running coprocessor post-open hooks at 1731423203517 (+2 ms)Region opened successfully at 1731423203518 (+1 ms) 2024-11-12T14:53:23,519 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731423203477 2024-11-12T14:53:23,521 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T14:53:23,522 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T14:53:23,522 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:23,522 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,46407,1731423200469, state=OPEN 2024-11-12T14:53:23,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:53:23,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:53:23,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100346cc1080003, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:53:23,524 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:23,524 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:23,524 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:23,524 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:53:23,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-12T14:53:23,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,46407,1731423200469 in 201 msec 2024-11-12T14:53:23,528 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:53:23,528 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.HRegionServer(1482): Serving as 3945e1ca6f47,45215,1731423203444, RpcServer on 3945e1ca6f47/172.17.0.2:45215, sessionid=0x100346cc1080003 2024-11-12T14:53:23,528 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T14:53:23,528 DEBUG [RS:2;3945e1ca6f47:45215 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3945e1ca6f47,45215,1731423203444 2024-11-12T14:53:23,528 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,45215,1731423203444' 2024-11-12T14:53:23,528 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T14:53:23,528 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T14:53:23,528 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 362 msec 2024-11-12T14:53:23,529 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 3945e1ca6f47,36205,1731423200499 had 2 regions 2024-11-12T14:53:23,529 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T14:53:23,529 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T14:53:23,529 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T14:53:23,530 DEBUG [RS:2;3945e1ca6f47:45215 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3945e1ca6f47,45215,1731423203444 2024-11-12T14:53:23,530 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,45215,1731423203444' 2024-11-12T14:53:23,530 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T14:53:23,530 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,36205,1731423200499, splitWal=true, meta=true, isMeta: false 2024-11-12T14:53:23,530 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T14:53:23,530 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,36205,1731423200499-splitting dir is empty, no logs to split. 2024-11-12T14:53:23,530 INFO [PEWorker-3 {}] master.SplitWALManager(105): 3945e1ca6f47,36205,1731423200499 WAL count=0, meta=false 2024-11-12T14:53:23,531 DEBUG [RS:2;3945e1ca6f47:45215 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T14:53:23,531 INFO [RS:2;3945e1ca6f47:45215 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T14:53:23,531 INFO [RS:2;3945e1ca6f47:45215 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T14:53:23,532 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,36205,1731423200499-splitting dir is empty, no logs to split. 2024-11-12T14:53:23,532 INFO [PEWorker-3 {}] master.SplitWALManager(105): 3945e1ca6f47,36205,1731423200499 WAL count=0, meta=false 2024-11-12T14:53:23,532 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 3945e1ca6f47,36205,1731423200499 WAL splitting is done? wals=0, meta=false 2024-11-12T14:53:23,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731423201238, region=573725d9ded94d93e3ea87860c9c7bca, ASSIGN}] 2024-11-12T14:53:23,534 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731423201238, region=573725d9ded94d93e3ea87860c9c7bca, ASSIGN 2024-11-12T14:53:23,535 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731423201238, region=573725d9ded94d93e3ea87860c9c7bca, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:53:23,632 INFO [RS:2;3945e1ca6f47:45215 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:53:23,633 INFO [RS:2;3945e1ca6f47:45215 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3945e1ca6f47%2C45215%2C1731423203444, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,45215,1731423203444, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs, maxLogs=55 2024-11-12T14:53:23,653 INFO [RS:2;3945e1ca6f47:45215 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/WALs/3945e1ca6f47,45215,1731423203444/3945e1ca6f47%2C45215%2C1731423203444.1731423203633 2024-11-12T14:53:23,653 DEBUG [RS:2;3945e1ca6f47:45215 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T14:53:23,683 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T14:53:23,683 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T14:53:23,685 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T14:53:23,685 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T14:53:23,685 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T14:53:23,685 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T14:53:23,685 DEBUG [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(204): Hosts are {3945e1ca6f47=0} racks are {/default-rack=0} 2024-11-12T14:53:23,686 DEBUG [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T14:53:23,686 DEBUG [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T14:53:23,686 DEBUG [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T14:53:23,686 DEBUG [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T14:53:23,686 INFO [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T14:53:23,686 INFO [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T14:53:23,686 DEBUG [3945e1ca6f47:34965 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T14:53:23,687 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=573725d9ded94d93e3ea87860c9c7bca, regionState=OPENING, regionLocation=3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:23,688 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:36205 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:36205 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:53:23,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,36205,1731423200499, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,36205,1731423200499, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:36205 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:36205 2024-11-12T14:53:23,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,36205,1731423200499, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:53:23,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,36205,1731423200499, seqNum=-1 from cache 2024-11-12T14:53:23,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:36205 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:36205 2024-11-12T14:53:23,795 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:53:23,795 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,46407,1731423200469, seqNum=-1] 2024-11-12T14:53:23,795 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:53:23,798 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33939, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:53:23,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731423201238, region=573725d9ded94d93e3ea87860c9c7bca, ASSIGN because future has completed 2024-11-12T14:53:23,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 573725d9ded94d93e3ea87860c9c7bca, server=3945e1ca6f47,46407,1731423200469}] 2024-11-12T14:53:23,961 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:23,961 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 573725d9ded94d93e3ea87860c9c7bca, NAME => 'testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:53:23,962 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731423201238 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:23,962 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:53:23,962 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:23,962 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:23,964 INFO [StoreOpener-573725d9ded94d93e3ea87860c9c7bca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:23,965 INFO [StoreOpener-573725d9ded94d93e3ea87860c9c7bca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 573725d9ded94d93e3ea87860c9c7bca columnFamilyName family 2024-11-12T14:53:23,965 DEBUG [StoreOpener-573725d9ded94d93e3ea87860c9c7bca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:53:23,966 DEBUG [StoreFileOpener-573725d9ded94d93e3ea87860c9c7bca-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/family/b6222671e8864a4691b329bb215a5f30, delay=951 ms 2024-11-12T14:53:23,967 DEBUG [StoreOpener-573725d9ded94d93e3ea87860c9c7bca-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/family/b6222671e8864a4691b329bb215a5f30 2024-11-12T14:53:23,967 INFO [StoreOpener-573725d9ded94d93e3ea87860c9c7bca-1 {}] regionserver.HStore(327): Store=573725d9ded94d93e3ea87860c9c7bca/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:53:23,967 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:23,968 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:23,968 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:23,969 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:23,969 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:23,970 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:23,970 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 573725d9ded94d93e3ea87860c9c7bca; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71705407, jitterRate=0.06849382817745209}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:53:23,970 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:23,971 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 573725d9ded94d93e3ea87860c9c7bca: Running coprocessor pre-open hook at 1731423203963Writing region info on filesystem at 1731423203963Initializing all the Stores at 1731423203963Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423203963Cleaning up temporary data from old regions at 1731423203969 (+6 ms)Running coprocessor post-open hooks at 1731423203970 (+1 ms)Region opened successfully at 1731423203971 (+1 ms) 2024-11-12T14:53:23,972 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca., pid=11, masterSystemTime=1731423203956 2024-11-12T14:53:23,974 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:23,975 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:23,975 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=573725d9ded94d93e3ea87860c9c7bca, regionState=OPEN, openSeqNum=10, regionLocation=3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:23,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 573725d9ded94d93e3ea87860c9c7bca, server=3945e1ca6f47,46407,1731423200469 because future has completed 2024-11-12T14:53:23,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-12T14:53:23,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 573725d9ded94d93e3ea87860c9c7bca, server=3945e1ca6f47,46407,1731423200469 in 177 msec 2024-11-12T14:53:23,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-12T14:53:23,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731423201238, region=573725d9ded94d93e3ea87860c9c7bca, ASSIGN in 448 msec 2024-11-12T14:53:23,983 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 3945e1ca6f47,36205,1731423200499 after splitting done 2024-11-12T14:53:23,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 3945e1ca6f47,36205,1731423200499, splitWal=true, meta=true in 829 msec 2024-11-12T14:53:24,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T14:53:24,028 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:53:24,029 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:53:24,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:24,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:24,029 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T14:53:24,030 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=723323453, stopped=false 2024-11-12T14:53:24,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:53:24,030 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3945e1ca6f47,34965,1731423200423 2024-11-12T14:53:24,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100346cc1080003, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:53:24,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:53:24,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:53:24,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:24,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:24,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100346cc1080003, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:24,032 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:53:24,033 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:53:24,033 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45215-0x100346cc1080003, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:53:24,033 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:53:24,033 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:53:24,034 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:53:24,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:24,034 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3945e1ca6f47,46407,1731423200469' ***** 2024-11-12T14:53:24,034 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T14:53:24,034 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3945e1ca6f47,45215,1731423203444' ***** 2024-11-12T14:53:24,034 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T14:53:24,034 INFO [RS:2;3945e1ca6f47:45215 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T14:53:24,035 INFO [RS:0;3945e1ca6f47:46407 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T14:53:24,035 INFO [RS:0;3945e1ca6f47:46407 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.persistence 2024-11-12T14:53:24,035 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T14:53:24,035 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T14:53:24,035 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T14:53:24,035 INFO [RS:2;3945e1ca6f47:45215 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9108f3a4-7d61-cddf-cb1c-8ff790b5d0e0/bucket.persistence 2024-11-12T14:53:24,035 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T14:53:24,035 INFO [RS:0;3945e1ca6f47:46407 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T14:53:24,035 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T14:53:24,035 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T14:53:24,035 INFO [RS:2;3945e1ca6f47:45215 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T14:53:24,035 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T14:53:24,036 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T14:53:24,045 DEBUG [RS:0;3945e1ca6f47:46407 {}] bucket.BucketCache(1488): Thread RS:0;3945e1ca6f47:46407 started persisting bucket cache to file 2024-11-12T14:53:24,046 DEBUG [RS:2;3945e1ca6f47:45215 {}] bucket.BucketCache(1488): Thread RS:2;3945e1ca6f47:45215 started persisting bucket cache to file 2024-11-12T14:53:24,046 DEBUG [RS:0;3945e1ca6f47:46407 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T14:53:24,046 DEBUG [RS:0;3945e1ca6f47:46407 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T14:53:24,046 DEBUG [RS:2;3945e1ca6f47:45215 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T14:53:24,046 DEBUG [RS:2;3945e1ca6f47:45215 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T14:53:24,050 DEBUG [RS:0;3945e1ca6f47:46407 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T14:53:24,051 DEBUG [RS:0;3945e1ca6f47:46407 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731423204046 2024-11-12T14:53:24,051 DEBUG [RS:2;3945e1ca6f47:45215 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-12T14:53:24,051 DEBUG [RS:2;3945e1ca6f47:45215 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731423204046 2024-11-12T14:53:24,051 DEBUG [RS:0;3945e1ca6f47:46407 {}] bucket.BucketCache(1510): Thread RS:0;3945e1ca6f47:46407 finished persisting bucket cache to file, renaming 2024-11-12T14:53:24,051 DEBUG [RS:2;3945e1ca6f47:45215 {}] bucket.BucketCache(1510): Thread RS:2;3945e1ca6f47:45215 finished persisting bucket cache to file, renaming 2024-11-12T14:53:24,051 WARN [RS:2;3945e1ca6f47:45215 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-12T14:53:24,051 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T14:53:24,051 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T14:53:24,051 INFO [RS:2;3945e1ca6f47:45215 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T14:53:24,051 INFO [RS:0;3945e1ca6f47:46407 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T14:53:24,051 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T14:53:24,051 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T14:53:24,051 INFO [RS:2;3945e1ca6f47:45215 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T14:53:24,051 INFO [RS:0;3945e1ca6f47:46407 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T14:53:24,052 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.HRegionServer(959): stopping server 3945e1ca6f47,45215,1731423203444 2024-11-12T14:53:24,052 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(3091): Received CLOSE for 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:24,052 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:53:24,052 INFO [RS:2;3945e1ca6f47:45215 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;3945e1ca6f47:45215. 2024-11-12T14:53:24,052 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(959): stopping server 3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:24,052 DEBUG [RS:2;3945e1ca6f47:45215 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:53:24,052 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:53:24,052 DEBUG [RS:2;3945e1ca6f47:45215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:24,052 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T14:53:24,052 INFO [RS:0;3945e1ca6f47:46407 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3945e1ca6f47:46407. 2024-11-12T14:53:24,052 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 573725d9ded94d93e3ea87860c9c7bca, disabling compactions & flushes 2024-11-12T14:53:24,052 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:24,052 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.HRegionServer(976): stopping server 3945e1ca6f47,45215,1731423203444; all regions closed. 2024-11-12T14:53:24,053 DEBUG [RS:0;3945e1ca6f47:46407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:53:24,053 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:24,053 DEBUG [RS:0;3945e1ca6f47:46407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:24,053 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. after waiting 0 ms 2024-11-12T14:53:24,053 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:24,053 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T14:53:24,053 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T14:53:24,053 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T14:53:24,053 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T14:53:24,054 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-12T14:53:24,054 DEBUG [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 573725d9ded94d93e3ea87860c9c7bca=testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca.} 2024-11-12T14:53:24,054 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T14:53:24,054 DEBUG [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 573725d9ded94d93e3ea87860c9c7bca 2024-11-12T14:53:24,054 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T14:53:24,054 DEBUG [StoreFileCloser-573725d9ded94d93e3ea87860c9c7bca-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/family/b6222671e8864a4691b329bb215a5f30 2024-11-12T14:53:24,054 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T14:53:24,054 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T14:53:24,054 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T14:53:24,054 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T14:53:24,055 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-12T14:53:24,056 DEBUG [RS:2;3945e1ca6f47:45215 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs 2024-11-12T14:53:24,056 INFO [RS:2;3945e1ca6f47:45215 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C45215%2C1731423203444:(num 1731423203633) 2024-11-12T14:53:24,056 DEBUG [RS:2;3945e1ca6f47:45215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:24,056 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:53:24,056 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:53:24,056 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.ChoreService(370): Chore service for: regionserver/3945e1ca6f47:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T14:53:24,056 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T14:53:24,057 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T14:53:24,057 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T14:53:24,057 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:53:24,057 INFO [regionserver/3945e1ca6f47:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:53:24,057 INFO [RS:2;3945e1ca6f47:45215 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45215 2024-11-12T14:53:24,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100346cc1080003, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3945e1ca6f47,45215,1731423203444 2024-11-12T14:53:24,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:53:24,059 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:53:24,060 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3945e1ca6f47,45215,1731423203444] 2024-11-12T14:53:24,061 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/default/testBlockEvictionOnGracefulStop1731423201238/573725d9ded94d93e3ea87860c9c7bca/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-12T14:53:24,061 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:24,061 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 573725d9ded94d93e3ea87860c9c7bca: Waiting for close lock at 1731423204052Running coprocessor pre-close hooks at 1731423204052Disabling compacts and flushes for region at 1731423204052Disabling writes for close at 1731423204053 (+1 ms)Writing region close event to WAL at 1731423204054 (+1 ms)Running coprocessor post-close hooks at 1731423204061 (+7 ms)Closed at 1731423204061 2024-11-12T14:53:24,061 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3945e1ca6f47,45215,1731423203444 already deleted, retry=false 2024-11-12T14:53:24,061 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca. 2024-11-12T14:53:24,061 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3945e1ca6f47,45215,1731423203444 expired; onlineServers=1 2024-11-12T14:53:24,065 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6fb1818df7ab44229231c78ee59d3174 2024-11-12T14:53:24,077 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/info/6fb1818df7ab44229231c78ee59d3174 is 209, key is testBlockEvictionOnGracefulStop1731423201238,,1731423201242.573725d9ded94d93e3ea87860c9c7bca./info:regioninfo/1731423203975/Put/seqid=0 2024-11-12T14:53:24,077 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/info/6fb1818df7ab44229231c78ee59d3174 2024-11-12T14:53:24,079 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/.tmp/info/6fb1818df7ab44229231c78ee59d3174 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/6fb1818df7ab44229231c78ee59d3174 2024-11-12T14:53:24,080 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/6fb1818df7ab44229231c78ee59d3174, delay=903 ms 2024-11-12T14:53:24,081 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/6fb1818df7ab44229231c78ee59d3174, entries=8, sequenceid=19, filesize=6.6 K 2024-11-12T14:53:24,082 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 28ms, sequenceid=19, compaction requested=false 2024-11-12T14:53:24,082 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/88c67ecafde047de8696dc3921320d8c 2024-11-12T14:53:24,083 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/info/6fb1818df7ab44229231c78ee59d3174 2024-11-12T14:53:24,083 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/ns/f738e7dadb7448d9a5be4074e6899528 2024-11-12T14:53:24,083 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/table/881feece945349a1b0248a0b9de66fe8 2024-11-12T14:53:24,087 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-12T14:53:24,087 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T14:53:24,087 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T14:53:24,087 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731423204054Running coprocessor pre-close hooks at 1731423204054Disabling compacts and flushes for region at 1731423204054Disabling writes for close at 1731423204054Obtaining lock to block concurrent updates at 1731423204055 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731423204055Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731423204055Flushing stores of hbase:meta,,1.1588230740 at 1731423204055Flushing 1588230740/info: creating writer at 1731423204056 (+1 ms)Flushing 1588230740/info: appending metadata at 1731423204076 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731423204076Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45fd36d9: reopening flushed file at 1731423204079 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 28ms, sequenceid=19, compaction requested=false at 1731423204082 (+3 ms)Writing region close event to WAL at 1731423204084 (+2 ms)Running coprocessor post-close hooks at 1731423204087 (+3 ms)Closed at 1731423204087 2024-11-12T14:53:24,088 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T14:53:24,088 INFO [regionserver/3945e1ca6f47:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:53:24,097 INFO [regionserver/3945e1ca6f47:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:53:24,161 INFO [RS:2;3945e1ca6f47:45215 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:53:24,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100346cc1080003, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:24,161 INFO [RS:2;3945e1ca6f47:45215 {}] regionserver.HRegionServer(1031): Exiting; stopping=3945e1ca6f47,45215,1731423203444; zookeeper connection closed. 2024-11-12T14:53:24,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100346cc1080003, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:24,161 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@f9c4b96 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@f9c4b96 2024-11-12T14:53:24,254 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(976): stopping server 3945e1ca6f47,46407,1731423200469; all regions closed. 2024-11-12T14:53:24,256 DEBUG [RS:0;3945e1ca6f47:46407 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs 2024-11-12T14:53:24,256 INFO [RS:0;3945e1ca6f47:46407 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C46407%2C1731423200469.meta:.meta(num 1731423203482) 2024-11-12T14:53:24,258 DEBUG [RS:0;3945e1ca6f47:46407 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/oldWALs 2024-11-12T14:53:24,258 INFO [RS:0;3945e1ca6f47:46407 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C46407%2C1731423200469:(num 1731423200931) 2024-11-12T14:53:24,258 DEBUG [RS:0;3945e1ca6f47:46407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:53:24,258 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:53:24,258 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:53:24,258 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.ChoreService(370): Chore service for: regionserver/3945e1ca6f47:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T14:53:24,258 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:53:24,258 INFO [regionserver/3945e1ca6f47:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:53:24,259 INFO [RS:0;3945e1ca6f47:46407 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46407 2024-11-12T14:53:24,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3945e1ca6f47,46407,1731423200469 2024-11-12T14:53:24,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:53:24,260 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:53:24,261 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3945e1ca6f47,46407,1731423200469] 2024-11-12T14:53:24,262 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3945e1ca6f47,46407,1731423200469 already deleted, retry=false 2024-11-12T14:53:24,262 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3945e1ca6f47,46407,1731423200469 expired; onlineServers=0 2024-11-12T14:53:24,262 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3945e1ca6f47,34965,1731423200423' ***** 2024-11-12T14:53:24,262 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T14:53:24,262 INFO [M:0;3945e1ca6f47:34965 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:53:24,262 INFO [M:0;3945e1ca6f47:34965 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:53:24,262 DEBUG [M:0;3945e1ca6f47:34965 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T14:53:24,262 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T14:53:24,262 DEBUG [M:0;3945e1ca6f47:34965 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T14:53:24,262 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.small.0-1731423200737 {}] cleaner.HFileCleaner(306): Exit Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.small.0-1731423200737,5,FailOnTimeoutGroup] 2024-11-12T14:53:24,262 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.large.0-1731423200736 {}] cleaner.HFileCleaner(306): Exit Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.large.0-1731423200736,5,FailOnTimeoutGroup] 2024-11-12T14:53:24,263 INFO [M:0;3945e1ca6f47:34965 {}] hbase.ChoreService(370): Chore service for: master/3945e1ca6f47:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T14:53:24,263 INFO [M:0;3945e1ca6f47:34965 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:53:24,263 DEBUG [M:0;3945e1ca6f47:34965 {}] master.HMaster(1795): Stopping service threads 2024-11-12T14:53:24,263 INFO [M:0;3945e1ca6f47:34965 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T14:53:24,263 INFO [M:0;3945e1ca6f47:34965 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:53:24,263 INFO [M:0;3945e1ca6f47:34965 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:53:24,263 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T14:53:24,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T14:53:24,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:53:24,264 DEBUG [M:0;3945e1ca6f47:34965 {}] zookeeper.ZKUtil(347): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T14:53:24,264 WARN [M:0;3945e1ca6f47:34965 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T14:53:24,264 INFO [M:0;3945e1ca6f47:34965 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/.lastflushedseqids 2024-11-12T14:53:24,270 INFO [M:0;3945e1ca6f47:34965 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:53:24,270 INFO [M:0;3945e1ca6f47:34965 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T14:53:24,270 DEBUG [M:0;3945e1ca6f47:34965 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T14:53:24,270 INFO [M:0;3945e1ca6f47:34965 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:24,270 DEBUG [M:0;3945e1ca6f47:34965 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:24,270 DEBUG [M:0;3945e1ca6f47:34965 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T14:53:24,270 DEBUG [M:0;3945e1ca6f47:34965 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:24,270 INFO [M:0;3945e1ca6f47:34965 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-11-12T14:53:24,278 DEBUG [M:0;3945e1ca6f47:34965 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b28524a9036348e4926f514d6a1c3780 2024-11-12T14:53:24,295 DEBUG [M:0;3945e1ca6f47:34965 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b28524a9036348e4926f514d6a1c3780 is 82, key is hbase:meta,,1/info:regioninfo/1731423203521/Put/seqid=0 2024-11-12T14:53:24,295 INFO [M:0;3945e1ca6f47:34965 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b28524a9036348e4926f514d6a1c3780 2024-11-12T14:53:24,303 DEBUG [M:0;3945e1ca6f47:34965 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 93be6e7a57ee4007a7887796104e08de 2024-11-12T14:53:24,319 DEBUG [M:0;3945e1ca6f47:34965 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/93be6e7a57ee4007a7887796104e08de is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731423201621/Put/seqid=0 2024-11-12T14:53:24,320 INFO [M:0;3945e1ca6f47:34965 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/93be6e7a57ee4007a7887796104e08de 2024-11-12T14:53:24,326 DEBUG [M:0;3945e1ca6f47:34965 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4fce7d9a3d4c4f5cb9f11a5f98bc1d39 2024-11-12T14:53:24,339 DEBUG [M:0;3945e1ca6f47:34965 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4fce7d9a3d4c4f5cb9f11a5f98bc1d39 is 69, key is 3945e1ca6f47,45215,1731423203444/rs:state/1731423203483/Put/seqid=0 2024-11-12T14:53:24,339 INFO [M:0;3945e1ca6f47:34965 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4fce7d9a3d4c4f5cb9f11a5f98bc1d39 2024-11-12T14:53:24,341 INFO [M:0;3945e1ca6f47:34965 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4fce7d9a3d4c4f5cb9f11a5f98bc1d39 2024-11-12T14:53:24,341 DEBUG [M:0;3945e1ca6f47:34965 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b28524a9036348e4926f514d6a1c3780 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b28524a9036348e4926f514d6a1c3780 2024-11-12T14:53:24,342 INFO [M:0;3945e1ca6f47:34965 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b28524a9036348e4926f514d6a1c3780, entries=14, sequenceid=102, filesize=5.9 K 2024-11-12T14:53:24,343 DEBUG [M:0;3945e1ca6f47:34965 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/93be6e7a57ee4007a7887796104e08de as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/93be6e7a57ee4007a7887796104e08de 2024-11-12T14:53:24,344 INFO [M:0;3945e1ca6f47:34965 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/93be6e7a57ee4007a7887796104e08de, entries=11, sequenceid=102, filesize=6.5 K 2024-11-12T14:53:24,345 DEBUG [M:0;3945e1ca6f47:34965 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4fce7d9a3d4c4f5cb9f11a5f98bc1d39 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4fce7d9a3d4c4f5cb9f11a5f98bc1d39 2024-11-12T14:53:24,346 INFO [M:0;3945e1ca6f47:34965 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4fce7d9a3d4c4f5cb9f11a5f98bc1d39 2024-11-12T14:53:24,346 INFO [M:0;3945e1ca6f47:34965 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1986d7b-1e9c-e8cb-9cea-7750f60d3d9d/92148058-faa9-7103-44ad-f03bdfb44c23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4fce7d9a3d4c4f5cb9f11a5f98bc1d39, entries=3, sequenceid=102, filesize=5.3 K 2024-11-12T14:53:24,347 INFO [M:0;3945e1ca6f47:34965 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=102, compaction requested=false 2024-11-12T14:53:24,348 INFO [M:0;3945e1ca6f47:34965 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:53:24,348 DEBUG [M:0;3945e1ca6f47:34965 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731423204270Disabling compacts and flushes for region at 1731423204270Disabling writes for close at 1731423204270Obtaining lock to block concurrent updates at 1731423204270Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731423204270Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1731423204271 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731423204271Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731423204271Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731423204294 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731423204294Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731423204297 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731423204319 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731423204319Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731423204321 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731423204338 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731423204338Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69931871: reopening flushed file at 1731423204341 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3564d9f0: reopening flushed file at 1731423204343 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b83b055: reopening flushed file at 1731423204344 (+1 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=102, compaction requested=false at 1731423204347 (+3 ms)Writing region close event to WAL at 1731423204348 (+1 ms)Closed at 1731423204348 2024-11-12T14:53:24,349 INFO [M:0;3945e1ca6f47:34965 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T14:53:24,349 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:53:24,349 INFO [M:0;3945e1ca6f47:34965 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34965 2024-11-12T14:53:24,349 INFO [M:0;3945e1ca6f47:34965 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:53:24,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:24,361 INFO [RS:0;3945e1ca6f47:46407 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:53:24,361 INFO [RS:0;3945e1ca6f47:46407 {}] regionserver.HRegionServer(1031): Exiting; stopping=3945e1ca6f47,46407,1731423200469; zookeeper connection closed. 2024-11-12T14:53:24,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46407-0x100346cc1080001, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:24,362 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@134b3efc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@134b3efc 2024-11-12T14:53:24,362 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-12T14:53:24,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:24,451 INFO [M:0;3945e1ca6f47:34965 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:53:24,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34965-0x100346cc1080000, quorum=127.0.0.1:60193, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:53:24,454 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T14:53:24,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-12T14:53:24,455 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:53:24,461 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=69 (was 49) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 359), ProcessCount=11 (was 11), AvailableMemoryMB=7851 (was 7931)