2024-11-20 21:55:48,778 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 21:55:48,798 main DEBUG Took 0.017716 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 21:55:48,799 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 21:55:48,799 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 21:55:48,800 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 21:55:48,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,810 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 21:55:48,827 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,829 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,831 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,833 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,833 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,834 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,835 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,836 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,836 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,837 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,838 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,838 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,839 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,839 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,840 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,841 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,841 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,842 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,842 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,843 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,843 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:55:48,844 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,845 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 21:55:48,847 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:55:48,848 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 21:55:48,850 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 21:55:48,851 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 21:55:48,853 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 21:55:48,853 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 21:55:48,864 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 21:55:48,868 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 21:55:48,870 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 21:55:48,870 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 21:55:48,871 main DEBUG createAppenders(={Console}) 2024-11-20 21:55:48,872 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-20 21:55:48,872 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 21:55:48,872 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-20 21:55:48,873 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 21:55:48,874 main DEBUG OutputStream closed 2024-11-20 21:55:48,874 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 21:55:48,874 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 21:55:48,875 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-20 21:55:48,968 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 21:55:48,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 21:55:48,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 21:55:48,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 21:55:48,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 21:55:48,977 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 21:55:48,977 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 21:55:48,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 21:55:48,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 21:55:48,979 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 21:55:48,979 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 21:55:48,980 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 21:55:48,980 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 21:55:48,981 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 21:55:48,981 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 21:55:48,982 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 21:55:48,982 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 21:55:48,983 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 21:55:48,986 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 21:55:48,987 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-20 21:55:48,987 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 21:55:48,988 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-20T21:55:49,003 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-20 21:55:49,006 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 21:55:49,006 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T21:55:49,321 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a 2024-11-20T21:55:49,349 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=205, ProcessCount=11, AvailableMemoryMB=4829 2024-11-20T21:55:49,478 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T21:55:49,622 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:49,660 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/cluster_3b259b0e-1275-1e65-99af-267f267b03b9, deleteOnExit=true 2024-11-20T21:55:49,799 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/cluster_3b259b0e-1275-1e65-99af-267f267b03b9/zookeeper_0, clientPort=51960, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/cluster_3b259b0e-1275-1e65-99af-267f267b03b9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/cluster_3b259b0e-1275-1e65-99af-267f267b03b9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T21:55:49,819 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51960 2024-11-20T21:55:49,820 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:49,931 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39 with version=8 2024-11-20T21:55:49,932 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/hbase-staging 2024-11-20T21:55:50,042 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T21:55:50,305 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e1fae5a3a175:0 server-side Connection retries=45 2024-11-20T21:55:50,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:50,316 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:50,327 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:55:50,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:50,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:55:50,475 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T21:55:50,580 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:55:50,640 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T21:55:50,653 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T21:55:50,658 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:55:50,701 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 92912 (auto-detected) 2024-11-20T21:55:50,703 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T21:55:50,730 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43325 2024-11-20T21:55:50,759 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43325 connecting to ZooKeeper ensemble=127.0.0.1:51960 2024-11-20T21:55:50,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:433250x0, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:55:50,797 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43325-0x1005f226def0000 connected 2024-11-20T21:55:50,843 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:50,844 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:50,859 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:55:50,864 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39, hbase.cluster.distributed=false 2024-11-20T21:55:50,894 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:55:50,902 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43325 2024-11-20T21:55:50,903 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43325 2024-11-20T21:55:50,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43325 2024-11-20T21:55:50,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43325 2024-11-20T21:55:50,913 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43325 2024-11-20T21:55:51,035 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e1fae5a3a175:0 server-side Connection retries=45 2024-11-20T21:55:51,037 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:51,038 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:51,038 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:55:51,038 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:51,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:55:51,042 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T21:55:51,048 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:55:51,049 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35383 2024-11-20T21:55:51,053 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35383 connecting to ZooKeeper ensemble=127.0.0.1:51960 2024-11-20T21:55:51,054 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:51,055 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:51,062 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:353830x0, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:55:51,063 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:353830x0, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:55:51,066 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35383-0x1005f226def0001 connected 2024-11-20T21:55:51,072 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T21:55:51,092 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/bucket.cache 2024-11-20T21:55:51,101 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T21:55:51,103 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T21:55:51,105 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T21:55:51,117 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T21:55:51,121 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T21:55:51,136 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T21:55:51,136 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/bucket.persistence is complete. 2024-11-20T21:55:51,141 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T21:55:51,171 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T21:55:51,177 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T21:55:51,185 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:55:51,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35383 2024-11-20T21:55:51,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35383 2024-11-20T21:55:51,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35383 2024-11-20T21:55:51,196 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35383 2024-11-20T21:55:51,197 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35383 2024-11-20T21:55:51,215 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e1fae5a3a175:0 server-side Connection retries=45 2024-11-20T21:55:51,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:51,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:51,216 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:55:51,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:51,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:55:51,217 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T21:55:51,217 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:55:51,220 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39301 2024-11-20T21:55:51,222 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39301 connecting to ZooKeeper ensemble=127.0.0.1:51960 2024-11-20T21:55:51,223 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:51,223 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:51,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:393010x0, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:55:51,228 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:393010x0, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:55:51,228 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39301-0x1005f226def0002 connected 2024-11-20T21:55:51,229 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T21:55:51,237 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/bucket.cache 2024-11-20T21:55:51,238 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T21:55:51,238 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T21:55:51,238 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T21:55:51,239 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T21:55:51,239 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T21:55:51,239 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T21:55:51,240 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T21:55:51,240 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T21:55:51,240 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/bucket.persistence is complete. 2024-11-20T21:55:51,242 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T21:55:51,244 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:55:51,247 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39301 2024-11-20T21:55:51,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39301 2024-11-20T21:55:51,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39301 2024-11-20T21:55:51,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39301 2024-11-20T21:55:51,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39301 2024-11-20T21:55:51,272 DEBUG [M:0;e1fae5a3a175:43325 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e1fae5a3a175:43325 2024-11-20T21:55:51,273 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e1fae5a3a175,43325,1732139750117 2024-11-20T21:55:51,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:55:51,279 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:55:51,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:55:51,282 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e1fae5a3a175,43325,1732139750117 2024-11-20T21:55:51,315 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T21:55:51,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T21:55:51,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:51,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:51,315 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:51,317 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T21:55:51,318 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e1fae5a3a175,43325,1732139750117 from backup master directory 2024-11-20T21:55:51,321 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:55:51,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:55:51,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e1fae5a3a175,43325,1732139750117 2024-11-20T21:55:51,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:55:51,322 WARN [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:55:51,322 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e1fae5a3a175,43325,1732139750117 2024-11-20T21:55:51,324 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T21:55:51,326 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T21:55:51,339 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/hbase.id] with ID: 3ca04dfc-cbb8-4e1f-9083-118519ccf7a8 2024-11-20T21:55:51,339 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/.tmp/hbase.id 2024-11-20T21:55:51,348 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/hbase.id] 2024-11-20T21:55:51,405 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:51,414 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T21:55:51,444 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-20T21:55:51,449 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:51,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:51,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:51,488 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:55:51,492 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T21:55:51,560 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:55:51,570 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:55:51,672 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store 2024-11-20T21:55:51,696 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T21:55:51,699 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:51,701 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T21:55:51,701 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:55:51,701 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:55:51,704 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T21:55:51,704 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:55:51,704 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:55:51,705 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732139751700Disabling compacts and flushes for region at 1732139751700Disabling writes for close at 1732139751704 (+4 ms)Writing region close event to WAL at 1732139751704Closed at 1732139751704 2024-11-20T21:55:51,705 WARN [master/e1fae5a3a175:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/.initializing 2024-11-20T21:55:51,706 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/WALs/e1fae5a3a175,43325,1732139750117 2024-11-20T21:55:51,718 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:55:51,734 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e1fae5a3a175%2C43325%2C1732139750117, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/WALs/e1fae5a3a175,43325,1732139750117, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/oldWALs, maxLogs=10 2024-11-20T21:55:51,779 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/WALs/e1fae5a3a175,43325,1732139750117/e1fae5a3a175%2C43325%2C1732139750117.1732139751739 2024-11-20T21:55:51,780 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T21:55:51,781 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:55:51,781 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:51,785 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:51,787 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:51,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:51,877 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T21:55:51,883 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:51,885 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:51,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:51,892 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T21:55:51,892 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:51,893 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:55:51,894 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:51,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T21:55:51,900 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:51,900 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:55:51,901 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:51,906 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T21:55:51,907 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:51,908 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:55:51,909 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:51,914 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:51,915 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:51,921 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:51,922 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:51,926 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T21:55:51,929 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:51,937 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:55:51,939 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60012927, jitterRate=-0.10573770105838776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T21:55:51,951 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732139751800Initializing all the Stores at 1732139751801 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139751801Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139751802 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139751803 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139751803Cleaning up temporary data from old regions at 1732139751922 (+119 ms)Region opened successfully at 1732139751950 (+28 ms) 2024-11-20T21:55:51,953 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T21:55:52,026 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283fd397, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e1fae5a3a175/172.17.0.2:0 2024-11-20T21:55:52,109 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T21:55:52,135 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T21:55:52,135 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T21:55:52,139 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T21:55:52,140 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T21:55:52,146 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T21:55:52,146 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T21:55:52,176 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T21:55:52,186 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T21:55:52,188 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T21:55:52,191 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T21:55:52,195 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T21:55:52,196 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T21:55:52,198 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T21:55:52,203 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T21:55:52,204 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T21:55:52,206 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T21:55:52,207 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T21:55:52,227 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T21:55:52,230 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T21:55:52,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T21:55:52,243 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T21:55:52,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T21:55:52,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:52,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:52,244 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:52,246 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e1fae5a3a175,43325,1732139750117, sessionid=0x1005f226def0000, setting cluster-up flag (Was=false) 2024-11-20T21:55:52,258 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:52,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:52,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:52,271 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T21:55:52,274 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e1fae5a3a175,43325,1732139750117 2024-11-20T21:55:52,280 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:52,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:52,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:52,289 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T21:55:52,295 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e1fae5a3a175,43325,1732139750117 2024-11-20T21:55:52,318 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T21:55:52,365 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(746): ClusterId : 3ca04dfc-cbb8-4e1f-9083-118519ccf7a8 2024-11-20T21:55:52,367 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(746): ClusterId : 3ca04dfc-cbb8-4e1f-9083-118519ccf7a8 2024-11-20T21:55:52,377 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T21:55:52,377 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T21:55:52,387 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T21:55:52,387 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T21:55:52,388 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T21:55:52,388 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T21:55:52,395 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T21:55:52,395 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T21:55:52,396 DEBUG [RS:0;e1fae5a3a175:35383 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34601e22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e1fae5a3a175/172.17.0.2:0 2024-11-20T21:55:52,396 DEBUG [RS:1;e1fae5a3a175:39301 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d094804, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e1fae5a3a175/172.17.0.2:0 2024-11-20T21:55:52,424 DEBUG [RS:0;e1fae5a3a175:35383 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e1fae5a3a175:35383 2024-11-20T21:55:52,424 DEBUG [RS:1;e1fae5a3a175:39301 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e1fae5a3a175:39301 2024-11-20T21:55:52,427 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T21:55:52,427 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T21:55:52,428 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T21:55:52,428 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T21:55:52,428 DEBUG [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T21:55:52,428 DEBUG [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T21:55:52,431 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(2659): reportForDuty to master=e1fae5a3a175,43325,1732139750117 with port=39301, startcode=1732139751215 2024-11-20T21:55:52,435 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(2659): reportForDuty to master=e1fae5a3a175,43325,1732139750117 with port=35383, startcode=1732139750990 2024-11-20T21:55:52,445 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T21:55:52,447 DEBUG [RS:0;e1fae5a3a175:35383 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T21:55:52,449 DEBUG [RS:1;e1fae5a3a175:39301 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T21:55:52,460 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T21:55:52,470 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T21:55:52,477 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e1fae5a3a175,43325,1732139750117 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T21:55:52,487 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e1fae5a3a175:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:55:52,488 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e1fae5a3a175:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:55:52,488 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e1fae5a3a175:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:55:52,488 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e1fae5a3a175:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:55:52,489 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e1fae5a3a175:0, corePoolSize=10, maxPoolSize=10 2024-11-20T21:55:52,489 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,490 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e1fae5a3a175:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:55:52,490 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,514 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T21:55:52,516 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T21:55:52,524 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:52,525 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T21:55:52,553 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732139782552 2024-11-20T21:55:52,555 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T21:55:52,556 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T21:55:52,562 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41723, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T21:55:52,562 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36073, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T21:55:52,564 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T21:55:52,564 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T21:55:52,568 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T21:55:52,568 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T21:55:52,573 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T21:55:52,577 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,581 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T21:55:52,581 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T21:55:52,582 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T21:55:52,583 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T21:55:52,586 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T21:55:52,586 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T21:55:52,588 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e1fae5a3a175:0:becomeActiveMaster-HFileCleaner.large.0-1732139752587,5,FailOnTimeoutGroup] 2024-11-20T21:55:52,589 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e1fae5a3a175:0:becomeActiveMaster-HFileCleaner.small.0-1732139752588,5,FailOnTimeoutGroup] 2024-11-20T21:55:52,589 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,589 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T21:55:52,590 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,591 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,609 DEBUG [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-20T21:55:52,610 WARN [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-20T21:55:52,610 DEBUG [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-20T21:55:52,610 WARN [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-20T21:55:52,618 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T21:55:52,619 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39 2024-11-20T21:55:52,639 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:52,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T21:55:52,649 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T21:55:52,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:52,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:52,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T21:55:52,670 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T21:55:52,671 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:52,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:52,672 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T21:55:52,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T21:55:52,682 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:52,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:52,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T21:55:52,686 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T21:55:52,687 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:52,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:52,687 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T21:55:52,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740 2024-11-20T21:55:52,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740 2024-11-20T21:55:52,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T21:55:52,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T21:55:52,692 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T21:55:52,693 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T21:55:52,710 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:55:52,711 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69704187, jitterRate=0.03867332637310028}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T21:55:52,711 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(2659): reportForDuty to master=e1fae5a3a175,43325,1732139750117 with port=39301, startcode=1732139751215 2024-11-20T21:55:52,712 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(2659): reportForDuty to master=e1fae5a3a175,43325,1732139750117 with port=35383, startcode=1732139750990 2024-11-20T21:55:52,714 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e1fae5a3a175,39301,1732139751215 2024-11-20T21:55:52,716 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43325 {}] master.ServerManager(517): Registering regionserver=e1fae5a3a175,39301,1732139751215 2024-11-20T21:55:52,717 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732139752639Initializing all the Stores at 1732139752640 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139752640Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139752641 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139752641Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139752641Cleaning up temporary data from old regions at 1732139752692 (+51 ms)Region opened successfully at 1732139752717 (+25 ms) 2024-11-20T21:55:52,717 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T21:55:52,718 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T21:55:52,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T21:55:52,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T21:55:52,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T21:55:52,722 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T21:55:52,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732139752717Disabling compacts and flushes for region at 1732139752717Disabling writes for close at 1732139752718 (+1 ms)Writing region close event to WAL at 1732139752721 (+3 ms)Closed at 1732139752722 (+1 ms) 2024-11-20T21:55:52,725 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:52,725 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43325 {}] master.ServerManager(517): Registering regionserver=e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:52,725 DEBUG [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39 2024-11-20T21:55:52,725 DEBUG [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T21:55:52,726 DEBUG [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T21:55:52,726 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T21:55:52,726 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T21:55:52,729 DEBUG [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39 2024-11-20T21:55:52,729 DEBUG [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T21:55:52,729 DEBUG [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T21:55:52,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:55:52,735 DEBUG [RS:1;e1fae5a3a175:39301 {}] zookeeper.ZKUtil(111): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e1fae5a3a175,39301,1732139751215 2024-11-20T21:55:52,735 DEBUG [RS:0;e1fae5a3a175:35383 {}] zookeeper.ZKUtil(111): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:52,735 WARN [RS:1;e1fae5a3a175:39301 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:55:52,735 WARN [RS:0;e1fae5a3a175:35383 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:55:52,735 INFO [RS:1;e1fae5a3a175:39301 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:55:52,735 INFO [RS:0;e1fae5a3a175:35383 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:55:52,735 DEBUG [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:52,735 DEBUG [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,39301,1732139751215 2024-11-20T21:55:52,737 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e1fae5a3a175,39301,1732139751215] 2024-11-20T21:55:52,737 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e1fae5a3a175,35383,1732139750990] 2024-11-20T21:55:52,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T21:55:52,749 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T21:55:52,754 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T21:55:52,775 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T21:55:52,775 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T21:55:52,808 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T21:55:52,808 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T21:55:52,817 INFO [RS:0;e1fae5a3a175:35383 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T21:55:52,817 INFO [RS:1;e1fae5a3a175:39301 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T21:55:52,817 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,817 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,818 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T21:55:52,825 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T21:55:52,827 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,828 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,828 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,828 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,828 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,829 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,829 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T21:55:52,829 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e1fae5a3a175:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:55:52,829 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,829 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,830 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,830 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,830 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T21:55:52,830 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,830 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,830 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,830 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e1fae5a3a175:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:55:52,831 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,831 DEBUG [RS:1;e1fae5a3a175:39301 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e1fae5a3a175:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:55:52,831 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,831 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,831 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,831 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,831 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e1fae5a3a175:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:55:52,832 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,832 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,832 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,832 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,832 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,832 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:52,833 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e1fae5a3a175:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:55:52,833 DEBUG [RS:0;e1fae5a3a175:35383 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e1fae5a3a175:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:55:52,839 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,839 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,839 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,839 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,840 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,840 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,840 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,840 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,840 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,840 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,840 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,39301,1732139751215-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:55:52,840 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,35383,1732139750990-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:55:52,874 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T21:55:52,874 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T21:55:52,877 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,39301,1732139751215-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,877 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,35383,1732139750990-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,877 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,877 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,877 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.Replication(171): e1fae5a3a175,39301,1732139751215 started 2024-11-20T21:55:52,877 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.Replication(171): e1fae5a3a175,35383,1732139750990 started 2024-11-20T21:55:52,905 WARN [e1fae5a3a175:43325 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T21:55:52,910 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,910 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(1482): Serving as e1fae5a3a175,35383,1732139750990, RpcServer on e1fae5a3a175/172.17.0.2:35383, sessionid=0x1005f226def0001 2024-11-20T21:55:52,911 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T21:55:52,912 DEBUG [RS:0;e1fae5a3a175:35383 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:52,912 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1fae5a3a175,35383,1732139750990' 2024-11-20T21:55:52,912 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T21:55:52,914 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T21:55:52,914 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:52,914 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(1482): Serving as e1fae5a3a175,39301,1732139751215, RpcServer on e1fae5a3a175/172.17.0.2:39301, sessionid=0x1005f226def0002 2024-11-20T21:55:52,914 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T21:55:52,915 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T21:55:52,915 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T21:55:52,915 DEBUG [RS:1;e1fae5a3a175:39301 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e1fae5a3a175,39301,1732139751215 2024-11-20T21:55:52,915 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1fae5a3a175,39301,1732139751215' 2024-11-20T21:55:52,915 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T21:55:52,915 DEBUG [RS:0;e1fae5a3a175:35383 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:52,915 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1fae5a3a175,35383,1732139750990' 2024-11-20T21:55:52,915 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T21:55:52,915 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T21:55:52,916 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T21:55:52,916 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T21:55:52,916 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T21:55:52,916 DEBUG [RS:1;e1fae5a3a175:39301 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e1fae5a3a175,39301,1732139751215 2024-11-20T21:55:52,916 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1fae5a3a175,39301,1732139751215' 2024-11-20T21:55:52,916 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T21:55:52,916 DEBUG [RS:0;e1fae5a3a175:35383 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T21:55:52,916 INFO [RS:0;e1fae5a3a175:35383 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T21:55:52,917 INFO [RS:0;e1fae5a3a175:35383 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T21:55:52,917 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T21:55:52,917 DEBUG [RS:1;e1fae5a3a175:39301 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T21:55:52,918 INFO [RS:1;e1fae5a3a175:39301 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T21:55:52,918 INFO [RS:1;e1fae5a3a175:39301 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T21:55:53,026 INFO [RS:1;e1fae5a3a175:39301 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:55:53,028 INFO [RS:1;e1fae5a3a175:39301 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e1fae5a3a175%2C39301%2C1732139751215, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,39301,1732139751215, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs, maxLogs=55 2024-11-20T21:55:53,026 INFO [RS:0;e1fae5a3a175:35383 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:55:53,030 INFO [RS:0;e1fae5a3a175:35383 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e1fae5a3a175%2C35383%2C1732139750990, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,35383,1732139750990, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs, maxLogs=55 2024-11-20T21:55:53,054 INFO [RS:0;e1fae5a3a175:35383 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,35383,1732139750990/e1fae5a3a175%2C35383%2C1732139750990.1732139753032 2024-11-20T21:55:53,054 INFO [RS:1;e1fae5a3a175:39301 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,39301,1732139751215/e1fae5a3a175%2C39301%2C1732139751215.1732139753031 2024-11-20T21:55:53,054 DEBUG [RS:0;e1fae5a3a175:35383 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T21:55:53,055 DEBUG [RS:1;e1fae5a3a175:39301 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T21:55:53,158 DEBUG [e1fae5a3a175:43325 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T21:55:53,174 DEBUG [e1fae5a3a175:43325 {}] balancer.BalancerClusterState(204): Hosts are {e1fae5a3a175=0} racks are {/default-rack=0} 2024-11-20T21:55:53,179 DEBUG [e1fae5a3a175:43325 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T21:55:53,179 DEBUG [e1fae5a3a175:43325 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T21:55:53,179 DEBUG [e1fae5a3a175:43325 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T21:55:53,179 DEBUG [e1fae5a3a175:43325 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T21:55:53,179 INFO [e1fae5a3a175:43325 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T21:55:53,179 INFO [e1fae5a3a175:43325 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T21:55:53,179 DEBUG [e1fae5a3a175:43325 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T21:55:53,186 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:53,191 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e1fae5a3a175,35383,1732139750990, state=OPENING 2024-11-20T21:55:53,194 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T21:55:53,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:53,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:53,196 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:53,197 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:55:53,197 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:55:53,197 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:55:53,198 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T21:55:53,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e1fae5a3a175,35383,1732139750990}] 2024-11-20T21:55:53,387 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T21:55:53,391 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34817, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T21:55:53,403 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T21:55:53,403 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:55:53,405 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T21:55:53,406 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e1fae5a3a175%2C35383%2C1732139750990.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,35383,1732139750990, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs, maxLogs=55 2024-11-20T21:55:53,430 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,35383,1732139750990/e1fae5a3a175%2C35383%2C1732139750990.meta.1732139753409.meta 2024-11-20T21:55:53,431 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T21:55:53,431 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:55:53,433 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T21:55:53,436 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T21:55:53,441 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T21:55:53,445 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T21:55:53,447 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:53,447 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T21:55:53,447 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T21:55:53,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T21:55:53,451 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T21:55:53,451 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:53,452 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:53,452 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T21:55:53,454 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T21:55:53,454 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:53,454 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:53,455 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T21:55:53,456 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T21:55:53,456 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:53,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:53,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T21:55:53,459 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T21:55:53,459 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:53,459 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:53,460 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T21:55:53,461 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740 2024-11-20T21:55:53,463 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740 2024-11-20T21:55:53,464 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T21:55:53,464 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T21:55:53,464 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T21:55:53,467 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T21:55:53,468 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69917694, jitterRate=0.04185482859611511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T21:55:53,468 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T21:55:53,470 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732139753448Writing region info on filesystem at 1732139753448Initializing all the Stores at 1732139753449 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139753449Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139753449Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139753449Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139753449Cleaning up temporary data from old regions at 1732139753464 (+15 ms)Running coprocessor post-open hooks at 1732139753468 (+4 ms)Region opened successfully at 1732139753470 (+2 ms) 2024-11-20T21:55:53,479 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732139753375 2024-11-20T21:55:53,496 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:53,506 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e1fae5a3a175,35383,1732139750990, state=OPEN 2024-11-20T21:55:53,508 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T21:55:53,508 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T21:55:53,510 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:55:53,511 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:55:53,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:55:53,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:55:53,511 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:55:53,511 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:55:53,513 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:53,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T21:55:53,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e1fae5a3a175,35383,1732139750990 in 314 msec 2024-11-20T21:55:53,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T21:55:53,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 794 msec 2024-11-20T21:55:53,540 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T21:55:53,540 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T21:55:53,593 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:55:53,595 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e1fae5a3a175,35383,1732139750990, seqNum=-1] 2024-11-20T21:55:53,636 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:55:53,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40935, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:55:53,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3080 sec 2024-11-20T21:55:53,698 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732139753698, completionTime=-1 2024-11-20T21:55:53,702 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T21:55:53,703 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T21:55:53,771 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T21:55:53,772 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732139813771 2024-11-20T21:55:53,772 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732139873772 2024-11-20T21:55:53,772 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 69 msec 2024-11-20T21:55:53,784 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T21:55:53,816 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,43325,1732139750117-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:53,824 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,43325,1732139750117-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:53,825 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,43325,1732139750117-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:53,828 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e1fae5a3a175:43325, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:53,829 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:53,851 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:53,854 DEBUG [master/e1fae5a3a175:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T21:55:53,907 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.584sec 2024-11-20T21:55:53,915 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T21:55:53,917 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T21:55:53,918 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T21:55:53,919 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T21:55:53,919 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T21:55:53,920 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,43325,1732139750117-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:55:53,921 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,43325,1732139750117-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T21:55:53,940 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T21:55:53,941 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T21:55:53,947 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,43325,1732139750117-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:53,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8eba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:55:53,996 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T21:55:53,996 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T21:55:54,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e1fae5a3a175,43325,-1 for getting cluster id 2024-11-20T21:55:54,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:55:54,023 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3ca04dfc-cbb8-4e1f-9083-118519ccf7a8' 2024-11-20T21:55:54,029 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:55:54,029 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3ca04dfc-cbb8-4e1f-9083-118519ccf7a8" 2024-11-20T21:55:54,030 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad5d814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:55:54,030 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e1fae5a3a175,43325,-1] 2024-11-20T21:55:54,034 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:55:54,039 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:55:54,041 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:55:54,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201cd73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:55:54,046 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:55:54,079 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e1fae5a3a175,35383,1732139750990, seqNum=-1] 2024-11-20T21:55:54,080 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:55:54,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:55:54,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e1fae5a3a175,43325,1732139750117 2024-11-20T21:55:54,134 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T21:55:54,138 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e1fae5a3a175,43325,1732139750117 2024-11-20T21:55:54,141 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13e52bd1 2024-11-20T21:55:54,142 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T21:55:54,145 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T21:55:54,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732139754128', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:55:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43325 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732139754128 2024-11-20T21:55:54,162 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732139754128 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T21:55:54,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732139754128" procId is: 4 2024-11-20T21:55:54,166 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:54,169 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732139754128 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T21:55:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:55:54,183 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732139754128-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8c0d53c34ef147d050d9d39dd1c6dd27, NAME => 'testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732139754128', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39 2024-11-20T21:55:54,193 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732139754128-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:54,194 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732139754128-pool-0 {}] regionserver.HRegion(1722): Closing 8c0d53c34ef147d050d9d39dd1c6dd27, disabling compactions & flushes 2024-11-20T21:55:54,194 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732139754128-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:54,194 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732139754128-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:54,194 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732139754128-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. after waiting 0 ms 2024-11-20T21:55:54,194 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732139754128-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:54,194 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732139754128-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:54,195 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732139754128-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8c0d53c34ef147d050d9d39dd1c6dd27: Waiting for close lock at 1732139754194Disabling compacts and flushes for region at 1732139754194Disabling writes for close at 1732139754194Writing region close event to WAL at 1732139754194Closed at 1732139754194 2024-11-20T21:55:54,196 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732139754128 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T21:55:54,201 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732139754196"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732139754196"}]},"ts":"1732139754196"} 2024-11-20T21:55:54,205 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T21:55:54,206 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732139754128 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T21:55:54,209 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732139754128","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732139754206"}]},"ts":"1732139754206"} 2024-11-20T21:55:54,213 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732139754128, state=ENABLING in hbase:meta 2024-11-20T21:55:54,214 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {e1fae5a3a175=0} racks are {/default-rack=0} 2024-11-20T21:55:54,215 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T21:55:54,215 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T21:55:54,215 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T21:55:54,215 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T21:55:54,215 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T21:55:54,215 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T21:55:54,215 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T21:55:54,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732139754128, region=8c0d53c34ef147d050d9d39dd1c6dd27, ASSIGN}] 2024-11-20T21:55:54,218 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732139754128, region=8c0d53c34ef147d050d9d39dd1c6dd27, ASSIGN 2024-11-20T21:55:54,219 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732139754128, region=8c0d53c34ef147d050d9d39dd1c6dd27, ASSIGN; state=OFFLINE, location=e1fae5a3a175,39301,1732139751215; forceNewPlan=false, retain=false 2024-11-20T21:55:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:55:54,372 INFO [e1fae5a3a175:43325 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T21:55:54,373 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8c0d53c34ef147d050d9d39dd1c6dd27, regionState=OPENING, regionLocation=e1fae5a3a175,39301,1732139751215 2024-11-20T21:55:54,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732139754128, region=8c0d53c34ef147d050d9d39dd1c6dd27, ASSIGN because future has completed 2024-11-20T21:55:54,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c0d53c34ef147d050d9d39dd1c6dd27, server=e1fae5a3a175,39301,1732139751215}] 2024-11-20T21:55:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:55:54,534 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T21:55:54,538 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36987, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T21:55:54,542 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:54,543 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8c0d53c34ef147d050d9d39dd1c6dd27, NAME => 'testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:55:54,544 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732139754128 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:54,544 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:54,544 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:54,544 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:54,546 INFO [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:54,550 INFO [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c0d53c34ef147d050d9d39dd1c6dd27 columnFamilyName family 2024-11-20T21:55:54,550 DEBUG [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:54,551 INFO [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] regionserver.HStore(327): Store=8c0d53c34ef147d050d9d39dd1c6dd27/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:55:54,551 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:54,553 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:54,553 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:54,553 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:54,553 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:54,554 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:54,559 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:55:54,560 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8c0d53c34ef147d050d9d39dd1c6dd27; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66463845, jitterRate=-0.009611532092094421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:55:54,560 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:54,561 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8c0d53c34ef147d050d9d39dd1c6dd27: Running coprocessor pre-open hook at 1732139754545Writing region info on filesystem at 1732139754545Initializing all the Stores at 1732139754545Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139754545Cleaning up temporary data from old regions at 1732139754553 (+8 ms)Running coprocessor post-open hooks at 1732139754560 (+7 ms)Region opened successfully at 1732139754561 (+1 ms) 2024-11-20T21:55:54,562 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27., pid=6, masterSystemTime=1732139754533 2024-11-20T21:55:54,565 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:54,566 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:54,566 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8c0d53c34ef147d050d9d39dd1c6dd27, regionState=OPEN, openSeqNum=2, regionLocation=e1fae5a3a175,39301,1732139751215 2024-11-20T21:55:54,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c0d53c34ef147d050d9d39dd1c6dd27, server=e1fae5a3a175,39301,1732139751215 because future has completed 2024-11-20T21:55:54,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T21:55:54,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8c0d53c34ef147d050d9d39dd1c6dd27, server=e1fae5a3a175,39301,1732139751215 in 194 msec 2024-11-20T21:55:54,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T21:55:54,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732139754128, region=8c0d53c34ef147d050d9d39dd1c6dd27, ASSIGN in 360 msec 2024-11-20T21:55:54,579 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732139754128 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T21:55:54,580 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732139754128","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732139754580"}]},"ts":"1732139754580"} 2024-11-20T21:55:54,584 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732139754128, state=ENABLED in hbase:meta 2024-11-20T21:55:54,586 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732139754128 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T21:55:54,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732139754128 in 431 msec 2024-11-20T21:55:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:55:54,809 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732139754128 get assigned. Timeout = 60000ms 2024-11-20T21:55:54,812 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732139754128 completed 2024-11-20T21:55:54,813 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:55:54,821 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732139754128 assigned to meta. Checking AM states. 2024-11-20T21:55:54,823 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:55:54,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732139754128 assigned. 2024-11-20T21:55:54,836 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732139754128', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27., hostname=e1fae5a3a175,39301,1732139751215, seqNum=2] 2024-11-20T21:55:54,838 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:55:54,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:55:54,856 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8c0d53c34ef147d050d9d39dd1c6dd27 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T21:55:54,898 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7404a1f5a20d460f8f16fada522af3f5 2024-11-20T21:55:54,950 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/.tmp/family/7404a1f5a20d460f8f16fada522af3f5 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T21:55:54,957 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/.tmp/family/7404a1f5a20d460f8f16fada522af3f5 2024-11-20T21:55:55,004 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/.tmp/family/7404a1f5a20d460f8f16fada522af3f5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/family/7404a1f5a20d460f8f16fada522af3f5 2024-11-20T21:55:55,006 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/family/7404a1f5a20d460f8f16fada522af3f5, delay=939 ms 2024-11-20T21:55:55,009 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/family/7404a1f5a20d460f8f16fada522af3f5, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T21:55:55,018 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 8c0d53c34ef147d050d9d39dd1c6dd27 in 161ms, sequenceid=6, compaction requested=false 2024-11-20T21:55:55,019 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732139754128' 2024-11-20T21:55:55,022 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8c0d53c34ef147d050d9d39dd1c6dd27: 2024-11-20T21:55:55,948 DEBUG [hfile-prefetch-1732139755007 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 7404a1f5a20d460f8f16fada522af3f5_0. 2024-11-20T21:55:55,951 DEBUG [hfile-prefetch-1732139755007 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 7404a1f5a20d460f8f16fada522af3f5_117. 2024-11-20T21:55:55,952 DEBUG [hfile-prefetch-1732139755007 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/family/7404a1f5a20d460f8f16fada522af3f5, with total blocks 2, and data blocks 1 2024-11-20T21:55:55,952 DEBUG [hfile-prefetch-1732139755007 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T21:55:55,952 DEBUG [hfile-prefetch-1732139755007 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/family/7404a1f5a20d460f8f16fada522af3f5, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T21:55:55,952 DEBUG [hfile-prefetch-1732139755007 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/family/7404a1f5a20d460f8f16fada522af3f5. Will sleep 100ms and try the verification again. 2024-11-20T21:55:56,022 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T21:55:56,022 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T21:55:56,022 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T21:55:56,022 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T21:55:56,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43325 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=8c0d53c34ef147d050d9d39dd1c6dd27, source=e1fae5a3a175,39301,1732139751215, destination=e1fae5a3a175,35383,1732139750990, warming up region on e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:56,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43325 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=8c0d53c34ef147d050d9d39dd1c6dd27, source=e1fae5a3a175,39301,1732139751215, destination=e1fae5a3a175,35383,1732139750990, running balancer 2024-11-20T21:55:56,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35383 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43325 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732139754128, region=8c0d53c34ef147d050d9d39dd1c6dd27, REOPEN/MOVE 2024-11-20T21:55:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35383 {}] regionserver.HRegion(7855): Warmup {ENCODED => 8c0d53c34ef147d050d9d39dd1c6dd27, NAME => 'testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:55:56,038 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732139754128, region=8c0d53c34ef147d050d9d39dd1c6dd27, REOPEN/MOVE 2024-11-20T21:55:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35383 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:56,038 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T21:55:56,038 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732139756022 2024-11-20T21:55:56,038 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T21:55:56,038 INFO [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,039 INFO [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c0d53c34ef147d050d9d39dd1c6dd27 columnFamilyName family 2024-11-20T21:55:56,040 DEBUG [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:56,040 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8c0d53c34ef147d050d9d39dd1c6dd27, regionState=CLOSING, regionLocation=e1fae5a3a175,39301,1732139751215 2024-11-20T21:55:56,043 DEBUG [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/family/7404a1f5a20d460f8f16fada522af3f5 2024-11-20T21:55:56,043 INFO [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] regionserver.HStore(327): Store=8c0d53c34ef147d050d9d39dd1c6dd27/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:55:56,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732139754128, region=8c0d53c34ef147d050d9d39dd1c6dd27, REOPEN/MOVE because future has completed 2024-11-20T21:55:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35383 {}] regionserver.HRegion(1722): Closing 8c0d53c34ef147d050d9d39dd1c6dd27, disabling compactions & flushes 2024-11-20T21:55:56,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35383 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35383 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35383 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. after waiting 0 ms 2024-11-20T21:55:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35383 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,044 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-20T21:55:56,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8c0d53c34ef147d050d9d39dd1c6dd27, server=e1fae5a3a175,39301,1732139751215}] 2024-11-20T21:55:56,048 DEBUG [StoreFileCloser-8c0d53c34ef147d050d9d39dd1c6dd27-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/family/7404a1f5a20d460f8f16fada522af3f5 2024-11-20T21:55:56,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35383 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35383 {}] regionserver.HRegion(1676): Region close journal for 8c0d53c34ef147d050d9d39dd1c6dd27: Waiting for close lock at 1732139756044Disabling compacts and flushes for region at 1732139756044Disabling writes for close at 1732139756044Writing region close event to WAL at 1732139756049 (+5 ms)Closed at 1732139756049 2024-11-20T21:55:56,051 WARN [hfile-prefetch-1732139755007 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/family/7404a1f5a20d460f8f16fada522af3f5, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-20T21:55:56,052 DEBUG [hfile-prefetch-1732139755007 {}] hfile.PrefetchExecutor(121): Prefetch completed for 7404a1f5a20d460f8f16fada522af3f5 2024-11-20T21:55:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43325 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-20T21:55:56,204 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,204 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-20T21:55:56,205 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 8c0d53c34ef147d050d9d39dd1c6dd27, disabling compactions & flushes 2024-11-20T21:55:56,206 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,206 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,206 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. after waiting 0 ms 2024-11-20T21:55:56,206 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,208 DEBUG [StoreFileCloser-8c0d53c34ef147d050d9d39dd1c6dd27-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 7404a1f5a20d460f8f16fada522af3f5, starting offset: 0, end offset: 9223372036854775807 2024-11-20T21:55:56,210 DEBUG [StoreFileCloser-8c0d53c34ef147d050d9d39dd1c6dd27-family-1 {}] bucket.BucketCache(856): removed key 7404a1f5a20d460f8f16fada522af3f5_0 from back map with offset lock 2094080 in the evict process 2024-11-20T21:55:56,229 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T21:55:56,230 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,230 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 8c0d53c34ef147d050d9d39dd1c6dd27: Waiting for close lock at 1732139756205Running coprocessor pre-close hooks at 1732139756205Disabling compacts and flushes for region at 1732139756205Disabling writes for close at 1732139756206 (+1 ms)Writing region close event to WAL at 1732139756211 (+5 ms)Running coprocessor post-close hooks at 1732139756229 (+18 ms)Closed at 1732139756230 (+1 ms) 2024-11-20T21:55:56,231 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 8c0d53c34ef147d050d9d39dd1c6dd27 move to e1fae5a3a175,35383,1732139750990 record at close sequenceid=6 2024-11-20T21:55:56,237 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,238 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8c0d53c34ef147d050d9d39dd1c6dd27, regionState=CLOSED 2024-11-20T21:55:56,238 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T21:55:56,238 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T21:55:56,239 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T21:55:56,239 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T21:55:56,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8c0d53c34ef147d050d9d39dd1c6dd27, server=e1fae5a3a175,39301,1732139751215 because future has completed 2024-11-20T21:55:56,243 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T21:55:56,243 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T21:55:56,243 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T21:55:56,243 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T21:55:56,252 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732139754128, region=8c0d53c34ef147d050d9d39dd1c6dd27, REOPEN/MOVE; state=CLOSED, location=e1fae5a3a175,35383,1732139750990; forceNewPlan=false, retain=false 2024-11-20T21:55:56,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T21:55:56,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 8c0d53c34ef147d050d9d39dd1c6dd27, server=e1fae5a3a175,39301,1732139751215 in 201 msec 2024-11-20T21:55:56,274 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T21:55:56,274 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732139756239 2024-11-20T21:55:56,274 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T21:55:56,281 WARN [Thread-92 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedReader.ensureOpen(BufferedReader.java:123) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:321) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-20T21:55:56,281 ERROR [bucket-cache-persister {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2024-11-20T21:55:56,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T21:55:56,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732139756243 2024-11-20T21:55:56,282 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T21:55:56,403 INFO [e1fae5a3a175:43325 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T21:55:56,405 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8c0d53c34ef147d050d9d39dd1c6dd27, regionState=OPENING, regionLocation=e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:56,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732139754128, region=8c0d53c34ef147d050d9d39dd1c6dd27, REOPEN/MOVE because future has completed 2024-11-20T21:55:56,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c0d53c34ef147d050d9d39dd1c6dd27, server=e1fae5a3a175,35383,1732139750990}] 2024-11-20T21:55:56,592 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,592 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 8c0d53c34ef147d050d9d39dd1c6dd27, NAME => 'testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:55:56,593 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732139754128 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,593 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:56,593 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,596 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,613 INFO [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,617 INFO [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c0d53c34ef147d050d9d39dd1c6dd27 columnFamilyName family 2024-11-20T21:55:56,619 DEBUG [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:56,623 DEBUG [StoreFileOpener-8c0d53c34ef147d050d9d39dd1c6dd27-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/family/7404a1f5a20d460f8f16fada522af3f5, delay=900 ms 2024-11-20T21:55:56,628 DEBUG [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/family/7404a1f5a20d460f8f16fada522af3f5 2024-11-20T21:55:56,628 INFO [StoreOpener-8c0d53c34ef147d050d9d39dd1c6dd27-1 {}] regionserver.HStore(327): Store=8c0d53c34ef147d050d9d39dd1c6dd27/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:55:56,629 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,630 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,632 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,632 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,632 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,642 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,644 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 8c0d53c34ef147d050d9d39dd1c6dd27; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63616775, jitterRate=-0.05203618109226227}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:55:56,644 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,652 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 8c0d53c34ef147d050d9d39dd1c6dd27: Running coprocessor pre-open hook at 1732139756597Writing region info on filesystem at 1732139756597Initializing all the Stores at 1732139756597Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139756597Cleaning up temporary data from old regions at 1732139756632 (+35 ms)Running coprocessor post-open hooks at 1732139756644 (+12 ms)Region opened successfully at 1732139756651 (+7 ms) 2024-11-20T21:55:56,657 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27., pid=9, masterSystemTime=1732139756574 2024-11-20T21:55:56,662 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8c0d53c34ef147d050d9d39dd1c6dd27, regionState=OPEN, openSeqNum=10, regionLocation=e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:56,662 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,662 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c0d53c34ef147d050d9d39dd1c6dd27, server=e1fae5a3a175,35383,1732139750990 because future has completed 2024-11-20T21:55:56,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-20T21:55:56,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 8c0d53c34ef147d050d9d39dd1c6dd27, server=e1fae5a3a175,35383,1732139750990 in 257 msec 2024-11-20T21:55:56,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732139754128, region=8c0d53c34ef147d050d9d39dd1c6dd27, REOPEN/MOVE in 645 msec 2024-11-20T21:55:56,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T21:55:56,692 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:55:56,693 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:55:56,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:55:56,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:55:56,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:55:56,701 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T21:55:56,701 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1058988911, stopped=false 2024-11-20T21:55:56,701 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e1fae5a3a175,43325,1732139750117 2024-11-20T21:55:56,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T21:55:56,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T21:55:56,703 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T21:55:56,703 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T21:55:56,703 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:56,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:56,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:56,704 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:55:56,705 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:55:56,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:55:56,705 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:55:56,705 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:55:56,705 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:55:56,706 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e1fae5a3a175,35383,1732139750990' ***** 2024-11-20T21:55:56,706 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T21:55:56,706 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e1fae5a3a175,39301,1732139751215' ***** 2024-11-20T21:55:56,707 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T21:55:56,707 INFO [RS:0;e1fae5a3a175:35383 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T21:55:56,707 INFO [RS:0;e1fae5a3a175:35383 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/bucket.persistence 2024-11-20T21:55:56,707 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T21:55:56,707 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T21:55:56,707 INFO [RS:1;e1fae5a3a175:39301 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T21:55:56,707 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T21:55:56,707 INFO [RS:1;e1fae5a3a175:39301 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/bucket.persistence 2024-11-20T21:55:56,707 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T21:55:56,707 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T21:55:56,708 INFO [RS:0;e1fae5a3a175:35383 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T21:55:56,708 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T21:55:56,708 INFO [RS:1;e1fae5a3a175:39301 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T21:55:56,708 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T21:55:56,708 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T21:55:56,718 DEBUG [RS:0;e1fae5a3a175:35383 {}] bucket.BucketCache(1487): Thread RS:0;e1fae5a3a175:35383 started persisting bucket cache to file 2024-11-20T21:55:56,718 DEBUG [RS:1;e1fae5a3a175:39301 {}] bucket.BucketCache(1487): Thread RS:1;e1fae5a3a175:39301 started persisting bucket cache to file 2024-11-20T21:55:56,718 DEBUG [RS:1;e1fae5a3a175:39301 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T21:55:56,718 DEBUG [RS:0;e1fae5a3a175:35383 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T21:55:56,718 DEBUG [RS:1;e1fae5a3a175:39301 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T21:55:56,718 DEBUG [RS:0;e1fae5a3a175:35383 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T21:55:56,730 DEBUG [RS:0;e1fae5a3a175:35383 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T21:55:56,730 DEBUG [RS:0;e1fae5a3a175:35383 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732139756718 2024-11-20T21:55:56,730 DEBUG [RS:0;e1fae5a3a175:35383 {}] bucket.BucketCache(1509): Thread RS:0;e1fae5a3a175:35383 finished persisting bucket cache to file, renaming 2024-11-20T21:55:56,730 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T21:55:56,731 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T21:55:56,731 INFO [RS:0;e1fae5a3a175:35383 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T21:55:56,731 INFO [RS:0;e1fae5a3a175:35383 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T21:55:56,731 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(3091): Received CLOSE for 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,737 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(959): stopping server e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:56,737 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:55:56,737 INFO [RS:0;e1fae5a3a175:35383 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e1fae5a3a175:35383. 2024-11-20T21:55:56,737 DEBUG [RS:0;e1fae5a3a175:35383 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:55:56,738 DEBUG [RS:0;e1fae5a3a175:35383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:55:56,737 ERROR [RS:1;e1fae5a3a175:39301 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:55:56,738 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T21:55:56,738 DEBUG [RS:1;e1fae5a3a175:39301 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T21:55:56,738 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T21:55:56,738 DEBUG [RS:1;e1fae5a3a175:39301 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732139756718 2024-11-20T21:55:56,738 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T21:55:56,738 DEBUG [RS:1;e1fae5a3a175:39301 {}] bucket.BucketCache(1509): Thread RS:1;e1fae5a3a175:39301 finished persisting bucket cache to file, renaming 2024-11-20T21:55:56,738 WARN [RS:1;e1fae5a3a175:39301 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-20T21:55:56,738 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T21:55:56,738 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T21:55:56,738 INFO [RS:1;e1fae5a3a175:39301 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T21:55:56,738 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T21:55:56,738 INFO [RS:1;e1fae5a3a175:39301 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T21:55:56,738 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(959): stopping server e1fae5a3a175,39301,1732139751215 2024-11-20T21:55:56,739 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:55:56,739 INFO [RS:1;e1fae5a3a175:39301 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e1fae5a3a175:39301. 2024-11-20T21:55:56,739 DEBUG [RS:1;e1fae5a3a175:39301 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:55:56,739 DEBUG [RS:1;e1fae5a3a175:39301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:55:56,739 WARN [Thread-96 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-20T21:55:56,739 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(976): stopping server e1fae5a3a175,39301,1732139751215; all regions closed. 2024-11-20T21:55:56,739 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T21:55:56,740 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8c0d53c34ef147d050d9d39dd1c6dd27, disabling compactions & flushes 2024-11-20T21:55:56,740 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,740 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,740 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. after waiting 0 ms 2024-11-20T21:55:56,740 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,741 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T21:55:56,741 DEBUG [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(1325): Online Regions={8c0d53c34ef147d050d9d39dd1c6dd27=testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27., 1588230740=hbase:meta,,1.1588230740} 2024-11-20T21:55:56,741 DEBUG [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8c0d53c34ef147d050d9d39dd1c6dd27 2024-11-20T21:55:56,742 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T21:55:56,742 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T21:55:56,742 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T21:55:56,742 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T21:55:56,742 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T21:55:56,742 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T21:55:56,743 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-20T21:55:56,747 DEBUG [StoreFileCloser-8c0d53c34ef147d050d9d39dd1c6dd27-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/family/7404a1f5a20d460f8f16fada522af3f5 2024-11-20T21:55:56,753 INFO [regionserver/e1fae5a3a175:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:55:56,753 INFO [regionserver/e1fae5a3a175:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:55:56,758 DEBUG [RS:1;e1fae5a3a175:39301 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs 2024-11-20T21:55:56,758 INFO [RS:1;e1fae5a3a175:39301 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e1fae5a3a175%2C39301%2C1732139751215:(num 1732139753031) 2024-11-20T21:55:56,758 DEBUG [RS:1;e1fae5a3a175:39301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:55:56,758 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:55:56,759 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:55:56,759 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.ChoreService(370): Chore service for: regionserver/e1fae5a3a175:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T21:55:56,759 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T21:55:56,759 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T21:55:56,759 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T21:55:56,760 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:55:56,760 INFO [RS:1;e1fae5a3a175:39301 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39301 2024-11-20T21:55:56,762 INFO [regionserver/e1fae5a3a175:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:55:56,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:55:56,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e1fae5a3a175,39301,1732139751215 2024-11-20T21:55:56,766 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:55:56,767 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e1fae5a3a175,39301,1732139751215] 2024-11-20T21:55:56,769 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e1fae5a3a175,39301,1732139751215 already deleted, retry=false 2024-11-20T21:55:56,769 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e1fae5a3a175,39301,1732139751215 expired; onlineServers=1 2024-11-20T21:55:56,770 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnRegionMove1732139754128/8c0d53c34ef147d050d9d39dd1c6dd27/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T21:55:56,770 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,771 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8c0d53c34ef147d050d9d39dd1c6dd27: Waiting for close lock at 1732139756737Running coprocessor pre-close hooks at 1732139756740 (+3 ms)Disabling compacts and flushes for region at 1732139756740Disabling writes for close at 1732139756740Writing region close event to WAL at 1732139756747 (+7 ms)Running coprocessor post-close hooks at 1732139756770 (+23 ms)Closed at 1732139756770 2024-11-20T21:55:56,771 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27. 2024-11-20T21:55:56,791 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on aa3c792d19b14fec8fe1e98202a0c54d 2024-11-20T21:55:56,810 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/info/aa3c792d19b14fec8fe1e98202a0c54d is 205, key is testBlockEvictionOnRegionMove1732139754128,,1732139754147.8c0d53c34ef147d050d9d39dd1c6dd27./info:regioninfo/1732139756662/Put/seqid=0 2024-11-20T21:55:56,813 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/info/aa3c792d19b14fec8fe1e98202a0c54d 2024-11-20T21:55:56,833 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 40d43f4c191c40c5b0493b69d5eb9993 2024-11-20T21:55:56,841 INFO [regionserver/e1fae5a3a175:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T21:55:56,841 INFO [regionserver/e1fae5a3a175:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T21:55:56,856 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/ns/40d43f4c191c40c5b0493b69d5eb9993 is 43, key is default/ns:d/1732139753659/Put/seqid=0 2024-11-20T21:55:56,857 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/ns/40d43f4c191c40c5b0493b69d5eb9993 2024-11-20T21:55:56,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:55:56,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39301-0x1005f226def0002, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:55:56,874 INFO [RS:1;e1fae5a3a175:39301 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:55:56,874 INFO [RS:1;e1fae5a3a175:39301 {}] regionserver.HRegionServer(1031): Exiting; stopping=e1fae5a3a175,39301,1732139751215; zookeeper connection closed. 2024-11-20T21:55:56,878 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a40133f462084667ab20f992263dbbc2 2024-11-20T21:55:56,878 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@78bf850e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@78bf850e 2024-11-20T21:55:56,897 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/table/a40133f462084667ab20f992263dbbc2 is 78, key is testBlockEvictionOnRegionMove1732139754128/table:state/1732139754580/Put/seqid=0 2024-11-20T21:55:56,898 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/table/a40133f462084667ab20f992263dbbc2 2024-11-20T21:55:56,903 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/info/aa3c792d19b14fec8fe1e98202a0c54d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/info/aa3c792d19b14fec8fe1e98202a0c54d 2024-11-20T21:55:56,905 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/info/aa3c792d19b14fec8fe1e98202a0c54d, delay=961 ms 2024-11-20T21:55:56,908 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/info/aa3c792d19b14fec8fe1e98202a0c54d, entries=15, sequenceid=15, filesize=7.6 K 2024-11-20T21:55:56,908 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/ns/40d43f4c191c40c5b0493b69d5eb9993 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/ns/40d43f4c191c40c5b0493b69d5eb9993 2024-11-20T21:55:56,911 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/ns/40d43f4c191c40c5b0493b69d5eb9993, delay=959 ms 2024-11-20T21:55:56,915 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/ns/40d43f4c191c40c5b0493b69d5eb9993, entries=2, sequenceid=15, filesize=5.0 K 2024-11-20T21:55:56,916 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/table/a40133f462084667ab20f992263dbbc2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/table/a40133f462084667ab20f992263dbbc2 2024-11-20T21:55:56,920 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/table/a40133f462084667ab20f992263dbbc2, delay=916 ms 2024-11-20T21:55:56,922 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/table/a40133f462084667ab20f992263dbbc2, entries=2, sequenceid=15, filesize=5.3 K 2024-11-20T21:55:56,923 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 181ms, sequenceid=15, compaction requested=false 2024-11-20T21:55:56,923 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T21:55:56,942 DEBUG [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T21:55:56,945 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/info/aa3c792d19b14fec8fe1e98202a0c54d 2024-11-20T21:55:56,946 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/ns/40d43f4c191c40c5b0493b69d5eb9993 2024-11-20T21:55:56,948 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/table/a40133f462084667ab20f992263dbbc2 2024-11-20T21:55:56,964 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-20T21:55:56,965 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T21:55:56,966 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T21:55:56,966 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732139756742Running coprocessor pre-close hooks at 1732139756742Disabling compacts and flushes for region at 1732139756742Disabling writes for close at 1732139756742Obtaining lock to block concurrent updates at 1732139756743 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732139756743Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732139756744 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732139756745 (+1 ms)Flushing 1588230740/info: creating writer at 1732139756746 (+1 ms)Flushing 1588230740/info: appending metadata at 1732139756808 (+62 ms)Flushing 1588230740/info: closing flushed file at 1732139756808Flushing 1588230740/ns: creating writer at 1732139756817 (+9 ms)Flushing 1588230740/ns: appending metadata at 1732139756854 (+37 ms)Flushing 1588230740/ns: closing flushed file at 1732139756854Flushing 1588230740/table: creating writer at 1732139756861 (+7 ms)Flushing 1588230740/table: appending metadata at 1732139756896 (+35 ms)Flushing 1588230740/table: closing flushed file at 1732139756896Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@369fc24c: reopening flushed file at 1732139756902 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50b49a10: reopening flushed file at 1732139756908 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d4ef9d0: reopening flushed file at 1732139756915 (+7 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 181ms, sequenceid=15, compaction requested=false at 1732139756923 (+8 ms)Writing region close event to WAL at 1732139756951 (+28 ms)Running coprocessor post-close hooks at 1732139756965 (+14 ms)Closed at 1732139756966 (+1 ms) 2024-11-20T21:55:56,966 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T21:55:57,142 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(976): stopping server e1fae5a3a175,35383,1732139750990; all regions closed. 2024-11-20T21:55:57,145 DEBUG [RS:0;e1fae5a3a175:35383 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs 2024-11-20T21:55:57,145 INFO [RS:0;e1fae5a3a175:35383 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e1fae5a3a175%2C35383%2C1732139750990.meta:.meta(num 1732139753409) 2024-11-20T21:55:57,147 DEBUG [RS:0;e1fae5a3a175:35383 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs 2024-11-20T21:55:57,147 INFO [RS:0;e1fae5a3a175:35383 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e1fae5a3a175%2C35383%2C1732139750990:(num 1732139753032) 2024-11-20T21:55:57,147 DEBUG [RS:0;e1fae5a3a175:35383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:55:57,147 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:55:57,148 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:55:57,148 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.ChoreService(370): Chore service for: regionserver/e1fae5a3a175:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T21:55:57,148 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:55:57,148 INFO [regionserver/e1fae5a3a175:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:55:57,148 INFO [RS:0;e1fae5a3a175:35383 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35383 2024-11-20T21:55:57,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:55:57,150 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e1fae5a3a175,35383,1732139750990 2024-11-20T21:55:57,150 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:55:57,151 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e1fae5a3a175,35383,1732139750990] 2024-11-20T21:55:57,154 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e1fae5a3a175,35383,1732139750990 already deleted, retry=false 2024-11-20T21:55:57,154 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e1fae5a3a175,35383,1732139750990 expired; onlineServers=0 2024-11-20T21:55:57,154 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e1fae5a3a175,43325,1732139750117' ***** 2024-11-20T21:55:57,154 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T21:55:57,154 INFO [M:0;e1fae5a3a175:43325 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:55:57,154 INFO [M:0;e1fae5a3a175:43325 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:55:57,154 DEBUG [M:0;e1fae5a3a175:43325 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T21:55:57,155 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T21:55:57,155 DEBUG [M:0;e1fae5a3a175:43325 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T21:55:57,155 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster-HFileCleaner.large.0-1732139752587 {}] cleaner.HFileCleaner(306): Exit Thread[master/e1fae5a3a175:0:becomeActiveMaster-HFileCleaner.large.0-1732139752587,5,FailOnTimeoutGroup] 2024-11-20T21:55:57,155 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster-HFileCleaner.small.0-1732139752588 {}] cleaner.HFileCleaner(306): Exit Thread[master/e1fae5a3a175:0:becomeActiveMaster-HFileCleaner.small.0-1732139752588,5,FailOnTimeoutGroup] 2024-11-20T21:55:57,155 INFO [M:0;e1fae5a3a175:43325 {}] hbase.ChoreService(370): Chore service for: master/e1fae5a3a175:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T21:55:57,155 INFO [M:0;e1fae5a3a175:43325 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:55:57,155 DEBUG [M:0;e1fae5a3a175:43325 {}] master.HMaster(1795): Stopping service threads 2024-11-20T21:55:57,155 INFO [M:0;e1fae5a3a175:43325 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T21:55:57,155 INFO [M:0;e1fae5a3a175:43325 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T21:55:57,156 INFO [M:0;e1fae5a3a175:43325 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T21:55:57,156 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T21:55:57,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T21:55:57,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,157 DEBUG [M:0;e1fae5a3a175:43325 {}] zookeeper.ZKUtil(347): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T21:55:57,157 WARN [M:0;e1fae5a3a175:43325 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T21:55:57,157 INFO [M:0;e1fae5a3a175:43325 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/.lastflushedseqids 2024-11-20T21:55:57,169 INFO [M:0;e1fae5a3a175:43325 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T21:55:57,169 INFO [M:0;e1fae5a3a175:43325 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T21:55:57,170 DEBUG [M:0;e1fae5a3a175:43325 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T21:55:57,170 INFO [M:0;e1fae5a3a175:43325 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:55:57,170 DEBUG [M:0;e1fae5a3a175:43325 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:55:57,170 DEBUG [M:0;e1fae5a3a175:43325 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T21:55:57,170 DEBUG [M:0;e1fae5a3a175:43325 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:55:57,170 INFO [M:0;e1fae5a3a175:43325 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-20T21:55:57,182 DEBUG [M:0;e1fae5a3a175:43325 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3d0ab295e2a0459b9b2506f54428e40e 2024-11-20T21:55:57,200 DEBUG [M:0;e1fae5a3a175:43325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3d0ab295e2a0459b9b2506f54428e40e is 82, key is hbase:meta,,1/info:regioninfo/1732139753495/Put/seqid=0 2024-11-20T21:55:57,201 INFO [M:0;e1fae5a3a175:43325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3d0ab295e2a0459b9b2506f54428e40e 2024-11-20T21:55:57,209 DEBUG [M:0;e1fae5a3a175:43325 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cf72837002024c8cb131fb386d285776 2024-11-20T21:55:57,222 DEBUG [M:0;e1fae5a3a175:43325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cf72837002024c8cb131fb386d285776 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732139754588/Put/seqid=0 2024-11-20T21:55:57,223 INFO [M:0;e1fae5a3a175:43325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cf72837002024c8cb131fb386d285776 2024-11-20T21:55:57,229 DEBUG [M:0;e1fae5a3a175:43325 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 74b1a14946b64200b80da799436546b1 2024-11-20T21:55:57,242 DEBUG [M:0;e1fae5a3a175:43325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74b1a14946b64200b80da799436546b1 is 69, key is e1fae5a3a175,35383,1732139750990/rs:state/1732139752726/Put/seqid=0 2024-11-20T21:55:57,242 INFO [M:0;e1fae5a3a175:43325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74b1a14946b64200b80da799436546b1 2024-11-20T21:55:57,244 DEBUG [M:0;e1fae5a3a175:43325 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3d0ab295e2a0459b9b2506f54428e40e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3d0ab295e2a0459b9b2506f54428e40e 2024-11-20T21:55:57,246 INFO [M:0;e1fae5a3a175:43325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3d0ab295e2a0459b9b2506f54428e40e, entries=8, sequenceid=84, filesize=5.5 K 2024-11-20T21:55:57,246 DEBUG [M:0;e1fae5a3a175:43325 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cf72837002024c8cb131fb386d285776 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cf72837002024c8cb131fb386d285776 2024-11-20T21:55:57,248 INFO [M:0;e1fae5a3a175:43325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cf72837002024c8cb131fb386d285776, entries=9, sequenceid=84, filesize=6.5 K 2024-11-20T21:55:57,248 DEBUG [M:0;e1fae5a3a175:43325 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74b1a14946b64200b80da799436546b1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/74b1a14946b64200b80da799436546b1 2024-11-20T21:55:57,250 INFO [M:0;e1fae5a3a175:43325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/74b1a14946b64200b80da799436546b1, entries=2, sequenceid=84, filesize=5.1 K 2024-11-20T21:55:57,251 INFO [M:0;e1fae5a3a175:43325 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 80ms, sequenceid=84, compaction requested=false 2024-11-20T21:55:57,252 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:55:57,252 INFO [RS:0;e1fae5a3a175:35383 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:55:57,252 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35383-0x1005f226def0001, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:55:57,252 INFO [RS:0;e1fae5a3a175:35383 {}] regionserver.HRegionServer(1031): Exiting; stopping=e1fae5a3a175,35383,1732139750990; zookeeper connection closed. 2024-11-20T21:55:57,252 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@375fd94b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@375fd94b 2024-11-20T21:55:57,253 INFO [M:0;e1fae5a3a175:43325 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:55:57,253 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-20T21:55:57,253 DEBUG [M:0;e1fae5a3a175:43325 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732139757169Disabling compacts and flushes for region at 1732139757169Disabling writes for close at 1732139757170 (+1 ms)Obtaining lock to block concurrent updates at 1732139757170Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732139757170Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1732139757171 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732139757171Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732139757171Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732139757199 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732139757199Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732139757203 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732139757221 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732139757221Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732139757224 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732139757241 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732139757241Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20d40927: reopening flushed file at 1732139757244 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3857f8cc: reopening flushed file at 1732139757246 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fd9ef4: reopening flushed file at 1732139757248 (+2 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 80ms, sequenceid=84, compaction requested=false at 1732139757251 (+3 ms)Writing region close event to WAL at 1732139757252 (+1 ms)Closed at 1732139757252 2024-11-20T21:55:57,254 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:55:57,254 INFO [M:0;e1fae5a3a175:43325 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T21:55:57,255 INFO [M:0;e1fae5a3a175:43325 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43325 2024-11-20T21:55:57,255 INFO [M:0;e1fae5a3a175:43325 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:55:57,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:55:57,356 INFO [M:0;e1fae5a3a175:43325 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:55:57,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43325-0x1005f226def0000, quorum=127.0.0.1:51960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:55:57,361 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T21:55:57,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T21:55:57,389 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:57,397 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732139756623 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732139756911 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/e1fae5a3a175:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/e1fae5a3a175:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/e1fae5a3a175:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732139755007 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/e1fae5a3a175:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732139756905 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=253 (was 205) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3885 (was 4829) 2024-11-20T21:55:57,404 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=253, ProcessCount=11, AvailableMemoryMB=3885 2024-11-20T21:55:57,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/hadoop-log-dir so I do NOT create it in target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b 2024-11-20T21:55:57,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/hadoop-tmp-dir so I do NOT create it in target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b 2024-11-20T21:55:57,404 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:57,407 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/cluster_7dc2813b-669c-624e-ff98-2f8b0bb8e524, deleteOnExit=true 2024-11-20T21:55:57,411 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/cluster_7dc2813b-669c-624e-ff98-2f8b0bb8e524/zookeeper_0, clientPort=57332, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/cluster_7dc2813b-669c-624e-ff98-2f8b0bb8e524/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/cluster_7dc2813b-669c-624e-ff98-2f8b0bb8e524/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T21:55:57,412 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57332 2024-11-20T21:55:57,423 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39 with version=8 2024-11-20T21:55:57,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/hbase-staging 2024-11-20T21:55:57,426 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e1fae5a3a175:0 server-side Connection retries=45 2024-11-20T21:55:57,426 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:57,426 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:57,426 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:55:57,426 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:57,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:55:57,427 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T21:55:57,427 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:55:57,428 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43385 2024-11-20T21:55:57,429 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43385 connecting to ZooKeeper ensemble=127.0.0.1:57332 2024-11-20T21:55:57,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:433850x0, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:55:57,434 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43385-0x1005f228bd00000 connected 2024-11-20T21:55:57,456 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:57,456 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:57,457 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:55:57,457 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39, hbase.cluster.distributed=false 2024-11-20T21:55:57,460 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:55:57,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43385 2024-11-20T21:55:57,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43385 2024-11-20T21:55:57,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43385 2024-11-20T21:55:57,466 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43385 2024-11-20T21:55:57,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43385 2024-11-20T21:55:57,491 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e1fae5a3a175:0 server-side Connection retries=45 2024-11-20T21:55:57,491 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:57,491 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:57,492 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:55:57,492 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:57,492 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:55:57,492 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T21:55:57,492 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:55:57,493 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39423 2024-11-20T21:55:57,496 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39423 connecting to ZooKeeper ensemble=127.0.0.1:57332 2024-11-20T21:55:57,497 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:57,497 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:57,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:394230x0, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:55:57,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:394230x0, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:55:57,504 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39423-0x1005f228bd00001 connected 2024-11-20T21:55:57,505 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T21:55:57,509 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.cache 2024-11-20T21:55:57,510 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T21:55:57,510 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T21:55:57,510 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T21:55:57,513 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T21:55:57,513 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T21:55:57,514 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T21:55:57,514 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.persistence is complete. 2024-11-20T21:55:57,514 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T21:55:57,517 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T21:55:57,518 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T21:55:57,520 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:55:57,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39423 2024-11-20T21:55:57,526 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39423 2024-11-20T21:55:57,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39423 2024-11-20T21:55:57,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39423 2024-11-20T21:55:57,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39423 2024-11-20T21:55:57,550 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e1fae5a3a175:0 server-side Connection retries=45 2024-11-20T21:55:57,550 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:57,550 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:57,550 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:55:57,550 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:55:57,550 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:55:57,550 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T21:55:57,551 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:55:57,551 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38757 2024-11-20T21:55:57,553 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38757 connecting to ZooKeeper ensemble=127.0.0.1:57332 2024-11-20T21:55:57,553 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:57,553 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:57,556 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387570x0, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:55:57,557 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38757-0x1005f228bd00002 connected 2024-11-20T21:55:57,557 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:55:57,557 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T21:55:57,559 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.cache 2024-11-20T21:55:57,559 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T21:55:57,559 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T21:55:57,560 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T21:55:57,560 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T21:55:57,560 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T21:55:57,560 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T21:55:57,560 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.persistence is complete. 2024-11-20T21:55:57,560 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T21:55:57,561 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T21:55:57,562 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T21:55:57,563 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:55:57,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38757 2024-11-20T21:55:57,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38757 2024-11-20T21:55:57,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38757 2024-11-20T21:55:57,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38757 2024-11-20T21:55:57,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38757 2024-11-20T21:55:57,582 DEBUG [M:0;e1fae5a3a175:43385 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e1fae5a3a175:43385 2024-11-20T21:55:57,583 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e1fae5a3a175,43385,1732139757425 2024-11-20T21:55:57,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:55:57,584 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:55:57,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:55:57,585 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e1fae5a3a175,43385,1732139757425 2024-11-20T21:55:57,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T21:55:57,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,587 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T21:55:57,587 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,587 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T21:55:57,588 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e1fae5a3a175,43385,1732139757425 from backup master directory 2024-11-20T21:55:57,589 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:55:57,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e1fae5a3a175,43385,1732139757425 2024-11-20T21:55:57,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:55:57,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:55:57,589 WARN [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:55:57,589 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e1fae5a3a175,43385,1732139757425 2024-11-20T21:55:57,590 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/hbase.id] with ID: 7142a613-be64-40b9-806c-ea0e1eb00ed4 2024-11-20T21:55:57,590 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/.tmp/hbase.id 2024-11-20T21:55:57,595 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/hbase.id] 2024-11-20T21:55:57,634 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:55:57,634 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T21:55:57,636 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-20T21:55:57,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,638 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,644 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:55:57,645 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T21:55:57,645 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:55:57,666 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store 2024-11-20T21:55:57,674 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:57,674 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T21:55:57,674 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:55:57,675 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:55:57,675 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T21:55:57,675 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:55:57,675 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:55:57,675 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732139757674Disabling compacts and flushes for region at 1732139757674Disabling writes for close at 1732139757675 (+1 ms)Writing region close event to WAL at 1732139757675Closed at 1732139757675 2024-11-20T21:55:57,675 WARN [master/e1fae5a3a175:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/.initializing 2024-11-20T21:55:57,675 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/WALs/e1fae5a3a175,43385,1732139757425 2024-11-20T21:55:57,686 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:55:57,691 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e1fae5a3a175%2C43385%2C1732139757425, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/WALs/e1fae5a3a175,43385,1732139757425, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/oldWALs, maxLogs=10 2024-11-20T21:55:57,716 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/WALs/e1fae5a3a175,43385,1732139757425/e1fae5a3a175%2C43385%2C1732139757425.1732139757692 2024-11-20T21:55:57,716 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T21:55:57,716 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:55:57,717 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:57,717 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:57,717 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:57,721 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:57,738 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T21:55:57,738 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:57,739 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:57,739 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:57,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T21:55:57,744 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:57,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:55:57,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:57,748 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T21:55:57,748 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:57,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:55:57,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:57,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T21:55:57,753 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:57,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:55:57,753 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:57,754 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:57,754 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:57,754 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:57,754 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:57,755 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T21:55:57,755 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:55:57,760 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:55:57,761 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69549911, jitterRate=0.03637443482875824}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T21:55:57,762 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732139757717Initializing all the Stores at 1732139757717Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139757718 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139757721 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139757721Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139757721Cleaning up temporary data from old regions at 1732139757754 (+33 ms)Region opened successfully at 1732139757761 (+7 ms) 2024-11-20T21:55:57,762 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T21:55:57,766 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c132c75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e1fae5a3a175/172.17.0.2:0 2024-11-20T21:55:57,767 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T21:55:57,767 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T21:55:57,767 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T21:55:57,768 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T21:55:57,768 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T21:55:57,768 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-20T21:55:57,768 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T21:55:57,771 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T21:55:57,772 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T21:55:57,773 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T21:55:57,773 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T21:55:57,774 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T21:55:57,775 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T21:55:57,775 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T21:55:57,777 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T21:55:57,778 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T21:55:57,779 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T21:55:57,780 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T21:55:57,782 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T21:55:57,783 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T21:55:57,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T21:55:57,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T21:55:57,786 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T21:55:57,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,786 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,787 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e1fae5a3a175,43385,1732139757425, sessionid=0x1005f228bd00000, setting cluster-up flag (Was=false) 2024-11-20T21:55:57,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,790 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,795 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T21:55:57,796 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e1fae5a3a175,43385,1732139757425 2024-11-20T21:55:57,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,798 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:57,802 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T21:55:57,803 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e1fae5a3a175,43385,1732139757425 2024-11-20T21:55:57,804 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T21:55:57,807 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T21:55:57,807 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T21:55:57,807 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T21:55:57,808 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e1fae5a3a175,43385,1732139757425 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T21:55:57,811 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e1fae5a3a175:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:55:57,811 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e1fae5a3a175:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:55:57,811 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e1fae5a3a175:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:55:57,811 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e1fae5a3a175:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:55:57,811 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e1fae5a3a175:0, corePoolSize=10, maxPoolSize=10 2024-11-20T21:55:57,812 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,812 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e1fae5a3a175:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:55:57,812 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,819 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T21:55:57,819 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T21:55:57,819 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732139787819 2024-11-20T21:55:57,820 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T21:55:57,820 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T21:55:57,820 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T21:55:57,820 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:57,820 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T21:55:57,820 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T21:55:57,820 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T21:55:57,820 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T21:55:57,821 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:57,824 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T21:55:57,824 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T21:55:57,824 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T21:55:57,825 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T21:55:57,825 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T21:55:57,825 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e1fae5a3a175:0:becomeActiveMaster-HFileCleaner.large.0-1732139757825,5,FailOnTimeoutGroup] 2024-11-20T21:55:57,829 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e1fae5a3a175:0:becomeActiveMaster-HFileCleaner.small.0-1732139757825,5,FailOnTimeoutGroup] 2024-11-20T21:55:57,829 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:57,829 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T21:55:57,829 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:57,830 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:57,850 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T21:55:57,850 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39 2024-11-20T21:55:57,858 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:57,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T21:55:57,870 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T21:55:57,870 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:57,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:57,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T21:55:57,874 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(746): ClusterId : 7142a613-be64-40b9-806c-ea0e1eb00ed4 2024-11-20T21:55:57,874 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T21:55:57,880 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T21:55:57,880 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T21:55:57,892 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(746): ClusterId : 7142a613-be64-40b9-806c-ea0e1eb00ed4 2024-11-20T21:55:57,892 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T21:55:57,892 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T21:55:57,893 DEBUG [RS:0;e1fae5a3a175:39423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2140af49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e1fae5a3a175/172.17.0.2:0 2024-11-20T21:55:57,897 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T21:55:57,897 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T21:55:57,903 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T21:55:57,903 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:57,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:57,903 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T21:55:57,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T21:55:57,904 DEBUG [RS:1;e1fae5a3a175:38757 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a42ffa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e1fae5a3a175/172.17.0.2:0 2024-11-20T21:55:57,931 DEBUG [RS:1;e1fae5a3a175:38757 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e1fae5a3a175:38757 2024-11-20T21:55:57,931 DEBUG [RS:0;e1fae5a3a175:39423 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e1fae5a3a175:39423 2024-11-20T21:55:57,931 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T21:55:57,931 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T21:55:57,931 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T21:55:57,931 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T21:55:57,931 DEBUG [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T21:55:57,932 DEBUG [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T21:55:57,933 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(2659): reportForDuty to master=e1fae5a3a175,43385,1732139757425 with port=38757, startcode=1732139757549 2024-11-20T21:55:57,933 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(2659): reportForDuty to master=e1fae5a3a175,43385,1732139757425 with port=39423, startcode=1732139757490 2024-11-20T21:55:57,933 DEBUG [RS:1;e1fae5a3a175:38757 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T21:55:57,933 DEBUG [RS:0;e1fae5a3a175:39423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T21:55:57,938 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T21:55:57,938 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:57,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:57,939 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T21:55:57,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T21:55:57,948 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36869, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T21:55:57,948 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54193, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T21:55:57,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:57,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:57,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T21:55:57,949 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e1fae5a3a175,38757,1732139757549 2024-11-20T21:55:57,949 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] master.ServerManager(517): Registering regionserver=e1fae5a3a175,38757,1732139757549 2024-11-20T21:55:57,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740 2024-11-20T21:55:57,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740 2024-11-20T21:55:57,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T21:55:57,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T21:55:57,951 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T21:55:57,951 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e1fae5a3a175,39423,1732139757490 2024-11-20T21:55:57,951 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] master.ServerManager(517): Registering regionserver=e1fae5a3a175,39423,1732139757490 2024-11-20T21:55:57,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T21:55:57,954 DEBUG [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39 2024-11-20T21:55:57,954 DEBUG [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T21:55:57,954 DEBUG [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T21:55:57,955 DEBUG [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39 2024-11-20T21:55:57,956 DEBUG [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T21:55:57,956 DEBUG [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T21:55:57,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:55:57,959 DEBUG [RS:0;e1fae5a3a175:39423 {}] zookeeper.ZKUtil(111): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e1fae5a3a175,39423,1732139757490 2024-11-20T21:55:57,959 WARN [RS:0;e1fae5a3a175:39423 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:55:57,959 INFO [RS:0;e1fae5a3a175:39423 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:55:57,959 DEBUG [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,39423,1732139757490 2024-11-20T21:55:57,959 DEBUG [RS:1;e1fae5a3a175:38757 {}] zookeeper.ZKUtil(111): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e1fae5a3a175,38757,1732139757549 2024-11-20T21:55:57,959 WARN [RS:1;e1fae5a3a175:38757 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:55:57,960 INFO [RS:1;e1fae5a3a175:38757 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:55:57,960 DEBUG [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,38757,1732139757549 2024-11-20T21:55:57,966 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e1fae5a3a175,38757,1732139757549] 2024-11-20T21:55:57,966 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e1fae5a3a175,39423,1732139757490] 2024-11-20T21:55:57,972 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T21:55:57,978 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:55:57,978 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64845478, jitterRate=-0.03372707962989807}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T21:55:57,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732139757858Initializing all the Stores at 1732139757859 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139757859Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139757859Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139757859Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139757859Cleaning up temporary data from old regions at 1732139757950 (+91 ms)Region opened successfully at 1732139757979 (+29 ms) 2024-11-20T21:55:57,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T21:55:57,980 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T21:55:57,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T21:55:57,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T21:55:57,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T21:55:57,985 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T21:55:57,990 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T21:55:57,990 INFO [RS:1;e1fae5a3a175:38757 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T21:55:57,990 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:57,991 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T21:55:57,992 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T21:55:57,992 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:57,992 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,992 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,992 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,993 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,993 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,993 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e1fae5a3a175:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:55:57,993 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,993 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,993 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T21:55:57,993 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,993 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732139757979Disabling compacts and flushes for region at 1732139757979Disabling writes for close at 1732139757980 (+1 ms)Writing region close event to WAL at 1732139757993 (+13 ms)Closed at 1732139757993 2024-11-20T21:55:57,993 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,993 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,994 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:57,994 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e1fae5a3a175:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:55:57,994 DEBUG [RS:1;e1fae5a3a175:38757 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e1fae5a3a175:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:55:57,994 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T21:55:57,994 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T21:55:57,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T21:55:57,996 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T21:55:57,996 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T21:55:57,997 INFO [RS:0;e1fae5a3a175:39423 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T21:55:57,997 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:57,998 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T21:55:58,001 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T21:55:58,002 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T21:55:58,002 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,002 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:58,002 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:58,002 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:58,002 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:58,002 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:58,002 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e1fae5a3a175:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:55:58,002 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:58,003 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:58,003 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:58,003 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:58,003 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:58,003 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:55:58,003 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e1fae5a3a175:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:55:58,003 DEBUG [RS:0;e1fae5a3a175:39423 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e1fae5a3a175:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:55:58,003 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,004 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,004 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,004 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,004 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,004 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,38757,1732139757549-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:55:58,013 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,013 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,013 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,013 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,014 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,014 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,39423,1732139757490-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:55:58,026 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T21:55:58,026 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,38757,1732139757549-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,026 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,026 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.Replication(171): e1fae5a3a175,38757,1732139757549 started 2024-11-20T21:55:58,036 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T21:55:58,036 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,39423,1732139757490-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,037 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,037 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.Replication(171): e1fae5a3a175,39423,1732139757490 started 2024-11-20T21:55:58,042 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,042 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(1482): Serving as e1fae5a3a175,38757,1732139757549, RpcServer on e1fae5a3a175/172.17.0.2:38757, sessionid=0x1005f228bd00002 2024-11-20T21:55:58,042 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T21:55:58,042 DEBUG [RS:1;e1fae5a3a175:38757 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e1fae5a3a175,38757,1732139757549 2024-11-20T21:55:58,042 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1fae5a3a175,38757,1732139757549' 2024-11-20T21:55:58,042 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T21:55:58,043 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T21:55:58,044 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T21:55:58,044 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T21:55:58,044 DEBUG [RS:1;e1fae5a3a175:38757 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e1fae5a3a175,38757,1732139757549 2024-11-20T21:55:58,044 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1fae5a3a175,38757,1732139757549' 2024-11-20T21:55:58,044 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T21:55:58,045 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T21:55:58,045 DEBUG [RS:1;e1fae5a3a175:38757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T21:55:58,045 INFO [RS:1;e1fae5a3a175:38757 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T21:55:58,045 INFO [RS:1;e1fae5a3a175:38757 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T21:55:58,060 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,060 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(1482): Serving as e1fae5a3a175,39423,1732139757490, RpcServer on e1fae5a3a175/172.17.0.2:39423, sessionid=0x1005f228bd00001 2024-11-20T21:55:58,061 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T21:55:58,061 DEBUG [RS:0;e1fae5a3a175:39423 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e1fae5a3a175,39423,1732139757490 2024-11-20T21:55:58,061 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1fae5a3a175,39423,1732139757490' 2024-11-20T21:55:58,061 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T21:55:58,062 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T21:55:58,063 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T21:55:58,063 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T21:55:58,063 DEBUG [RS:0;e1fae5a3a175:39423 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e1fae5a3a175,39423,1732139757490 2024-11-20T21:55:58,063 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1fae5a3a175,39423,1732139757490' 2024-11-20T21:55:58,063 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T21:55:58,065 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T21:55:58,066 DEBUG [RS:0;e1fae5a3a175:39423 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T21:55:58,066 INFO [RS:0;e1fae5a3a175:39423 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T21:55:58,066 INFO [RS:0;e1fae5a3a175:39423 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T21:55:58,146 INFO [RS:1;e1fae5a3a175:38757 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:55:58,147 INFO [RS:1;e1fae5a3a175:38757 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e1fae5a3a175%2C38757%2C1732139757549, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,38757,1732139757549, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs, maxLogs=55 2024-11-20T21:55:58,149 WARN [e1fae5a3a175:43385 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T21:55:58,166 INFO [RS:1;e1fae5a3a175:38757 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,38757,1732139757549/e1fae5a3a175%2C38757%2C1732139757549.1732139758149 2024-11-20T21:55:58,167 INFO [RS:0;e1fae5a3a175:39423 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:55:58,167 DEBUG [RS:1;e1fae5a3a175:38757 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T21:55:58,168 INFO [RS:0;e1fae5a3a175:39423 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e1fae5a3a175%2C39423%2C1732139757490, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,39423,1732139757490, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs, maxLogs=55 2024-11-20T21:55:58,186 INFO [RS:0;e1fae5a3a175:39423 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,39423,1732139757490/e1fae5a3a175%2C39423%2C1732139757490.1732139758168 2024-11-20T21:55:58,186 DEBUG [RS:0;e1fae5a3a175:39423 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T21:55:58,399 DEBUG [e1fae5a3a175:43385 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T21:55:58,399 DEBUG [e1fae5a3a175:43385 {}] balancer.BalancerClusterState(204): Hosts are {e1fae5a3a175=0} racks are {/default-rack=0} 2024-11-20T21:55:58,400 DEBUG [e1fae5a3a175:43385 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T21:55:58,401 DEBUG [e1fae5a3a175:43385 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T21:55:58,401 DEBUG [e1fae5a3a175:43385 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T21:55:58,401 DEBUG [e1fae5a3a175:43385 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T21:55:58,401 INFO [e1fae5a3a175:43385 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T21:55:58,401 INFO [e1fae5a3a175:43385 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T21:55:58,401 DEBUG [e1fae5a3a175:43385 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T21:55:58,401 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e1fae5a3a175,39423,1732139757490 2024-11-20T21:55:58,402 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e1fae5a3a175,39423,1732139757490, state=OPENING 2024-11-20T21:55:58,425 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T21:55:58,427 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:58,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:58,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:55:58,427 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T21:55:58,427 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:55:58,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e1fae5a3a175,39423,1732139757490}] 2024-11-20T21:55:58,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:55:58,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:55:58,581 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T21:55:58,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37565, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T21:55:58,589 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T21:55:58,589 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:55:58,590 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T21:55:58,591 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e1fae5a3a175%2C39423%2C1732139757490.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,39423,1732139757490, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs, maxLogs=55 2024-11-20T21:55:58,608 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,39423,1732139757490/e1fae5a3a175%2C39423%2C1732139757490.meta.1732139758592.meta 2024-11-20T21:55:58,608 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T21:55:58,609 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:55:58,609 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T21:55:58,609 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T21:55:58,609 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T21:55:58,609 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T21:55:58,610 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:58,610 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T21:55:58,610 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T21:55:58,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T21:55:58,611 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T21:55:58,611 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:58,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:58,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T21:55:58,612 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T21:55:58,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:58,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:58,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T21:55:58,613 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T21:55:58,613 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:58,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:58,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T21:55:58,614 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T21:55:58,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:58,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:55:58,614 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T21:55:58,615 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740 2024-11-20T21:55:58,615 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740 2024-11-20T21:55:58,615 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T21:55:58,615 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T21:55:58,615 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T21:55:58,616 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T21:55:58,616 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61705422, jitterRate=-0.08051756024360657}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T21:55:58,616 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T21:55:58,617 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732139758610Writing region info on filesystem at 1732139758610Initializing all the Stores at 1732139758610Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139758610Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139758610Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139758611 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732139758611Cleaning up temporary data from old regions at 1732139758615 (+4 ms)Running coprocessor post-open hooks at 1732139758617 (+2 ms)Region opened successfully at 1732139758617 2024-11-20T21:55:58,618 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732139758580 2024-11-20T21:55:58,621 DEBUG [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T21:55:58,621 INFO [RS_OPEN_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T21:55:58,621 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e1fae5a3a175,39423,1732139757490 2024-11-20T21:55:58,622 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e1fae5a3a175,39423,1732139757490, state=OPEN 2024-11-20T21:55:58,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:55:58,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:55:58,623 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:55:58,623 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e1fae5a3a175,39423,1732139757490 2024-11-20T21:55:58,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:55:58,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:55:58,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:55:58,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T21:55:58,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e1fae5a3a175,39423,1732139757490 in 196 msec 2024-11-20T21:55:58,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T21:55:58,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 631 msec 2024-11-20T21:55:58,629 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T21:55:58,629 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T21:55:58,633 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:55:58,633 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e1fae5a3a175,39423,1732139757490, seqNum=-1] 2024-11-20T21:55:58,633 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:55:58,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57609, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:55:58,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 834 msec 2024-11-20T21:55:58,641 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732139758641, completionTime=-1 2024-11-20T21:55:58,641 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T21:55:58,641 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T21:55:58,643 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T21:55:58,643 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732139818643 2024-11-20T21:55:58,644 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732139878644 2024-11-20T21:55:58,644 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-20T21:55:58,644 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,43385,1732139757425-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,644 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,43385,1732139757425-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,644 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,43385,1732139757425-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,644 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e1fae5a3a175:43385, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,644 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,645 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,647 DEBUG [master/e1fae5a3a175:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T21:55:58,649 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.060sec 2024-11-20T21:55:58,650 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T21:55:58,650 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T21:55:58,650 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T21:55:58,650 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T21:55:58,650 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T21:55:58,650 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,43385,1732139757425-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:55:58,650 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,43385,1732139757425-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T21:55:58,651 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T21:55:58,651 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T21:55:58,651 INFO [master/e1fae5a3a175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,43385,1732139757425-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:55:58,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be8a7fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:55:58,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e1fae5a3a175,43385,-1 for getting cluster id 2024-11-20T21:55:58,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:55:58,697 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7142a613-be64-40b9-806c-ea0e1eb00ed4' 2024-11-20T21:55:58,698 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:55:58,698 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7142a613-be64-40b9-806c-ea0e1eb00ed4" 2024-11-20T21:55:58,698 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b0f074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:55:58,699 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e1fae5a3a175,43385,-1] 2024-11-20T21:55:58,699 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:55:58,700 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:55:58,702 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:55:58,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2129cce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:55:58,705 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:55:58,707 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e1fae5a3a175,39423,1732139757490, seqNum=-1] 2024-11-20T21:55:58,707 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:55:58,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:55:58,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e1fae5a3a175,43385,1732139757425 2024-11-20T21:55:58,713 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T21:55:58,714 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is e1fae5a3a175,43385,1732139757425 2024-11-20T21:55:58,714 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@387ce449 2024-11-20T21:55:58,715 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T21:55:58,717 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T21:55:58,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732139758712', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:55:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732139758712 2024-11-20T21:55:58,720 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732139758712 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T21:55:58,720 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:58,720 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732139758712" procId is: 4 2024-11-20T21:55:58,721 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732139758712 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T21:55:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:55:58,732 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732139758712-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6377002468577638191cf24457a2724e, NAME => 'testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732139758712', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39 2024-11-20T21:55:58,739 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732139758712-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:58,739 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732139758712-pool-0 {}] regionserver.HRegion(1722): Closing 6377002468577638191cf24457a2724e, disabling compactions & flushes 2024-11-20T21:55:58,739 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732139758712-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:55:58,739 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732139758712-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:55:58,739 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732139758712-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. after waiting 0 ms 2024-11-20T21:55:58,740 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732139758712-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:55:58,740 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732139758712-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:55:58,740 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732139758712-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6377002468577638191cf24457a2724e: Waiting for close lock at 1732139758739Disabling compacts and flushes for region at 1732139758739Disabling writes for close at 1732139758739Writing region close event to WAL at 1732139758740 (+1 ms)Closed at 1732139758740 2024-11-20T21:55:58,741 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732139758712 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T21:55:58,741 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732139758741"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732139758741"}]},"ts":"1732139758741"} 2024-11-20T21:55:58,744 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T21:55:58,745 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732139758712 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T21:55:58,745 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732139758712","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732139758745"}]},"ts":"1732139758745"} 2024-11-20T21:55:58,747 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732139758712, state=ENABLING in hbase:meta 2024-11-20T21:55:58,747 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {e1fae5a3a175=0} racks are {/default-rack=0} 2024-11-20T21:55:58,748 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T21:55:58,748 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T21:55:58,748 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T21:55:58,748 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T21:55:58,748 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T21:55:58,748 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T21:55:58,748 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T21:55:58,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732139758712, region=6377002468577638191cf24457a2724e, ASSIGN}] 2024-11-20T21:55:58,750 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732139758712, region=6377002468577638191cf24457a2724e, ASSIGN 2024-11-20T21:55:58,751 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732139758712, region=6377002468577638191cf24457a2724e, ASSIGN; state=OFFLINE, location=e1fae5a3a175,38757,1732139757549; forceNewPlan=false, retain=false 2024-11-20T21:55:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:55:58,902 INFO [e1fae5a3a175:43385 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T21:55:58,902 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6377002468577638191cf24457a2724e, regionState=OPENING, regionLocation=e1fae5a3a175,38757,1732139757549 2024-11-20T21:55:58,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732139758712, region=6377002468577638191cf24457a2724e, ASSIGN because future has completed 2024-11-20T21:55:58,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6377002468577638191cf24457a2724e, server=e1fae5a3a175,38757,1732139757549}] 2024-11-20T21:55:58,948 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:55:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:55:59,061 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T21:55:59,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38375, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T21:55:59,068 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:55:59,069 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6377002468577638191cf24457a2724e, NAME => 'testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:55:59,070 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732139758712 6377002468577638191cf24457a2724e 2024-11-20T21:55:59,070 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:55:59,070 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6377002468577638191cf24457a2724e 2024-11-20T21:55:59,070 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6377002468577638191cf24457a2724e 2024-11-20T21:55:59,072 INFO [StoreOpener-6377002468577638191cf24457a2724e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 6377002468577638191cf24457a2724e 2024-11-20T21:55:59,078 INFO [StoreOpener-6377002468577638191cf24457a2724e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6377002468577638191cf24457a2724e columnFamilyName family 2024-11-20T21:55:59,078 DEBUG [StoreOpener-6377002468577638191cf24457a2724e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:55:59,078 INFO [StoreOpener-6377002468577638191cf24457a2724e-1 {}] regionserver.HStore(327): Store=6377002468577638191cf24457a2724e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:55:59,078 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6377002468577638191cf24457a2724e 2024-11-20T21:55:59,079 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e 2024-11-20T21:55:59,079 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e 2024-11-20T21:55:59,080 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6377002468577638191cf24457a2724e 2024-11-20T21:55:59,080 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6377002468577638191cf24457a2724e 2024-11-20T21:55:59,081 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6377002468577638191cf24457a2724e 2024-11-20T21:55:59,087 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:55:59,087 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6377002468577638191cf24457a2724e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72281529, jitterRate=0.07707871496677399}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:55:59,087 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6377002468577638191cf24457a2724e 2024-11-20T21:55:59,088 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6377002468577638191cf24457a2724e: Running coprocessor pre-open hook at 1732139759071Writing region info on filesystem at 1732139759071Initializing all the Stores at 1732139759071Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139759071Cleaning up temporary data from old regions at 1732139759080 (+9 ms)Running coprocessor post-open hooks at 1732139759087 (+7 ms)Region opened successfully at 1732139759088 (+1 ms) 2024-11-20T21:55:59,089 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e., pid=6, masterSystemTime=1732139759061 2024-11-20T21:55:59,092 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:55:59,093 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:55:59,093 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6377002468577638191cf24457a2724e, regionState=OPEN, openSeqNum=2, regionLocation=e1fae5a3a175,38757,1732139757549 2024-11-20T21:55:59,095 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e1fae5a3a175,38757,1732139757549, table=testBlockEvictionOnGracefulStop1732139758712, region=6377002468577638191cf24457a2724e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T21:55:59,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6377002468577638191cf24457a2724e, server=e1fae5a3a175,38757,1732139757549 because future has completed 2024-11-20T21:55:59,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T21:55:59,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6377002468577638191cf24457a2724e, server=e1fae5a3a175,38757,1732139757549 in 192 msec 2024-11-20T21:55:59,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T21:55:59,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732139758712, region=6377002468577638191cf24457a2724e, ASSIGN in 354 msec 2024-11-20T21:55:59,104 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732139758712 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T21:55:59,104 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732139758712","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732139759104"}]},"ts":"1732139759104"} 2024-11-20T21:55:59,107 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732139758712, state=ENABLED in hbase:meta 2024-11-20T21:55:59,108 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732139758712 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T21:55:59,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732139758712 in 390 msec 2024-11-20T21:55:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:55:59,347 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732139758712 completed 2024-11-20T21:55:59,347 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732139758712 get assigned. Timeout = 60000ms 2024-11-20T21:55:59,347 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:55:59,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732139758712 assigned to meta. Checking AM states. 2024-11-20T21:55:59,352 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:55:59,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732139758712 assigned. 2024-11-20T21:55:59,357 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732139758712', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e., hostname=e1fae5a3a175,38757,1732139757549, seqNum=2] 2024-11-20T21:55:59,358 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:55:59,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:55:59,366 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6377002468577638191cf24457a2724e 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T21:55:59,378 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on aff55f404a254870b8fa95d59f134e73 2024-11-20T21:55:59,395 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/.tmp/family/aff55f404a254870b8fa95d59f134e73 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T21:55:59,395 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/.tmp/family/aff55f404a254870b8fa95d59f134e73 2024-11-20T21:55:59,398 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/.tmp/family/aff55f404a254870b8fa95d59f134e73 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/family/aff55f404a254870b8fa95d59f134e73 2024-11-20T21:55:59,399 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/family/aff55f404a254870b8fa95d59f134e73, delay=905 ms 2024-11-20T21:55:59,400 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/family/aff55f404a254870b8fa95d59f134e73, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T21:55:59,401 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 6377002468577638191cf24457a2724e in 35ms, sequenceid=6, compaction requested=false 2024-11-20T21:55:59,401 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732139758712' 2024-11-20T21:55:59,401 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6377002468577638191cf24457a2724e: 2024-11-20T21:56:00,305 DEBUG [hfile-prefetch-1732139755007 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key aff55f404a254870b8fa95d59f134e73_0. 2024-11-20T21:56:00,306 DEBUG [hfile-prefetch-1732139755007 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key aff55f404a254870b8fa95d59f134e73_117. 2024-11-20T21:56:00,307 DEBUG [hfile-prefetch-1732139755007 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/family/aff55f404a254870b8fa95d59f134e73, with total blocks 2, and data blocks 1 2024-11-20T21:56:00,307 DEBUG [hfile-prefetch-1732139755007 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T21:56:00,307 DEBUG [hfile-prefetch-1732139755007 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/family/aff55f404a254870b8fa95d59f134e73, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T21:56:00,307 DEBUG [hfile-prefetch-1732139755007 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/family/aff55f404a254870b8fa95d59f134e73. Will sleep 100ms and try the verification again. 2024-11-20T21:56:00,316 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T21:56:00,316 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T21:56:00,316 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T21:56:00,316 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T21:56:00,319 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T21:56:00,319 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732139760316 2024-11-20T21:56:00,319 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T21:56:00,402 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;e1fae5a3a175:38757,5,FailOnTimeoutGroup] 2024-11-20T21:56:00,402 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e1fae5a3a175,38757,1732139757549' ***** 2024-11-20T21:56:00,402 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-20T21:56:00,402 INFO [RS:1;e1fae5a3a175:38757 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T21:56:00,403 INFO [RS:1;e1fae5a3a175:38757 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.persistence 2024-11-20T21:56:00,403 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T21:56:00,403 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T21:56:00,403 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T21:56:00,403 INFO [RS:1;e1fae5a3a175:38757 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T21:56:00,403 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T21:56:00,404 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T21:56:00,408 DEBUG [hfile-prefetch-1732139755007 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/family/aff55f404a254870b8fa95d59f134e73, with total blocks 2, and data blocks 1 2024-11-20T21:56:00,408 DEBUG [hfile-prefetch-1732139755007 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-20T21:56:00,408 DEBUG [hfile-prefetch-1732139755007 {}] bucket.BucketCache(2293): found block for file aff55f404a254870b8fa95d59f134e73 in the backing map. Acquiring read lock for offset 0 2024-11-20T21:56:00,408 DEBUG [hfile-prefetch-1732139755007 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/family/aff55f404a254870b8fa95d59f134e73 has now been fully cached. 2024-11-20T21:56:00,408 DEBUG [hfile-prefetch-1732139755007 {}] hfile.PrefetchExecutor(121): Prefetch completed for aff55f404a254870b8fa95d59f134e73 2024-11-20T21:56:00,411 INFO [regionserver/e1fae5a3a175:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:56:00,413 DEBUG [RS:1;e1fae5a3a175:38757 {}] bucket.BucketCache(1487): Thread RS:1;e1fae5a3a175:38757 started persisting bucket cache to file 2024-11-20T21:56:00,413 DEBUG [RS:1;e1fae5a3a175:38757 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T21:56:00,413 DEBUG [RS:1;e1fae5a3a175:38757 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-20T21:56:00,422 DEBUG [RS:1;e1fae5a3a175:38757 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-20T21:56:00,422 DEBUG [RS:1;e1fae5a3a175:38757 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732139760413 2024-11-20T21:56:00,422 DEBUG [RS:1;e1fae5a3a175:38757 {}] bucket.BucketCache(1509): Thread RS:1;e1fae5a3a175:38757 finished persisting bucket cache to file, renaming 2024-11-20T21:56:00,422 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T21:56:00,422 INFO [RS:1;e1fae5a3a175:38757 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T21:56:00,422 INFO [RS:1;e1fae5a3a175:38757 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T21:56:00,422 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(3091): Received CLOSE for 6377002468577638191cf24457a2724e 2024-11-20T21:56:00,423 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(959): stopping server e1fae5a3a175,38757,1732139757549 2024-11-20T21:56:00,423 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:56:00,423 INFO [RS:1;e1fae5a3a175:38757 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e1fae5a3a175:38757. 2024-11-20T21:56:00,423 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T21:56:00,423 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6377002468577638191cf24457a2724e, disabling compactions & flushes 2024-11-20T21:56:00,423 DEBUG [RS:1;e1fae5a3a175:38757 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:56:00,423 DEBUG [RS:1;e1fae5a3a175:38757 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:56:00,423 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:00,423 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:00,423 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T21:56:00,423 DEBUG [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(1325): Online Regions={6377002468577638191cf24457a2724e=testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e.} 2024-11-20T21:56:00,423 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. after waiting 0 ms 2024-11-20T21:56:00,424 DEBUG [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(1351): Waiting on 6377002468577638191cf24457a2724e 2024-11-20T21:56:00,424 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:00,431 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T21:56:00,431 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:00,431 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6377002468577638191cf24457a2724e: Waiting for close lock at 1732139760423Running coprocessor pre-close hooks at 1732139760423Disabling compacts and flushes for region at 1732139760423Disabling writes for close at 1732139760424 (+1 ms)Writing region close event to WAL at 1732139760425 (+1 ms)Running coprocessor post-close hooks at 1732139760431 (+6 ms)Closed at 1732139760431 2024-11-20T21:56:00,431 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:00,624 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(976): stopping server e1fae5a3a175,38757,1732139757549; all regions closed. 2024-11-20T21:56:00,627 DEBUG [RS:1;e1fae5a3a175:38757 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs 2024-11-20T21:56:00,627 INFO [RS:1;e1fae5a3a175:38757 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e1fae5a3a175%2C38757%2C1732139757549:(num 1732139758149) 2024-11-20T21:56:00,627 DEBUG [RS:1;e1fae5a3a175:38757 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:56:00,627 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:56:00,628 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:56:00,628 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.ChoreService(370): Chore service for: regionserver/e1fae5a3a175:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T21:56:00,628 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T21:56:00,628 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T21:56:00,628 INFO [regionserver/e1fae5a3a175:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:56:00,628 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T21:56:00,628 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:56:00,629 INFO [RS:1;e1fae5a3a175:38757 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38757 2024-11-20T21:56:00,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:56:00,634 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e1fae5a3a175,38757,1732139757549 2024-11-20T21:56:00,634 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:56:00,635 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e1fae5a3a175,38757,1732139757549] 2024-11-20T21:56:00,635 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T21:56:00,636 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T21:56:00,636 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e1fae5a3a175,38757,1732139757549 already deleted, retry=false 2024-11-20T21:56:00,636 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of e1fae5a3a175,38757,1732139757549 on e1fae5a3a175,43385,1732139757425 2024-11-20T21:56:00,638 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T21:56:00,638 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T21:56:00,638 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T21:56:00,638 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T21:56:00,641 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure e1fae5a3a175,38757,1732139757549, splitWal=true, meta=false 2024-11-20T21:56:00,643 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for e1fae5a3a175,38757,1732139757549 (carryingMeta=false) e1fae5a3a175,38757,1732139757549/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2ac679fc[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T21:56:00,643 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure e1fae5a3a175,38757,1732139757549, splitWal=true, meta=false 2024-11-20T21:56:00,644 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): e1fae5a3a175,38757,1732139757549 had 1 regions 2024-11-20T21:56:00,646 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure e1fae5a3a175,38757,1732139757549, splitWal=true, meta=false, isMeta: false 2024-11-20T21:56:00,646 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,38757,1732139757549-splitting 2024-11-20T21:56:00,646 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,38757,1732139757549-splitting dir is empty, no logs to split. 2024-11-20T21:56:00,646 INFO [PEWorker-2 {}] master.SplitWALManager(105): e1fae5a3a175,38757,1732139757549 WAL count=0, meta=false 2024-11-20T21:56:00,648 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,38757,1732139757549-splitting dir is empty, no logs to split. 2024-11-20T21:56:00,648 INFO [PEWorker-2 {}] master.SplitWALManager(105): e1fae5a3a175,38757,1732139757549 WAL count=0, meta=false 2024-11-20T21:56:00,648 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if e1fae5a3a175,38757,1732139757549 WAL splitting is done? wals=0, meta=false 2024-11-20T21:56:00,649 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for e1fae5a3a175,38757,1732139757549 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,38757,1732139757549-splitting does not exist 2024-11-20T21:56:00,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732139758712, region=6377002468577638191cf24457a2724e, ASSIGN}] 2024-11-20T21:56:00,651 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732139758712, region=6377002468577638191cf24457a2724e, ASSIGN 2024-11-20T21:56:00,652 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732139758712, region=6377002468577638191cf24457a2724e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T21:56:00,735 INFO [RS:1;e1fae5a3a175:38757 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:56:00,735 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:56:00,736 INFO [RS:1;e1fae5a3a175:38757 {}] regionserver.HRegionServer(1031): Exiting; stopping=e1fae5a3a175,38757,1732139757549; zookeeper connection closed. 2024-11-20T21:56:00,736 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38757-0x1005f228bd00002, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:56:00,736 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67db4b44 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67db4b44 2024-11-20T21:56:00,803 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=6377002468577638191cf24457a2724e, regionState=OPENING, regionLocation=e1fae5a3a175,39423,1732139757490 2024-11-20T21:56:00,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732139758712, region=6377002468577638191cf24457a2724e, ASSIGN because future has completed 2024-11-20T21:56:00,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6377002468577638191cf24457a2724e, server=e1fae5a3a175,39423,1732139757490}] 2024-11-20T21:56:00,919 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e1fae5a3a175:0 server-side Connection retries=45 2024-11-20T21:56:00,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:56:00,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:56:00,919 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:56:00,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:56:00,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:56:00,920 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T21:56:00,920 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:56:00,920 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37401 2024-11-20T21:56:00,922 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37401 connecting to ZooKeeper ensemble=127.0.0.1:57332 2024-11-20T21:56:00,922 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:56:00,922 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:56:00,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374010x0, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:56:00,925 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37401-0x1005f228bd00003 connected 2024-11-20T21:56:00,925 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37401-0x1005f228bd00003, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-20T21:56:00,925 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-20T21:56:00,926 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T21:56:00,926 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.cache 2024-11-20T21:56:00,926 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T21:56:00,927 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T21:56:00,927 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T21:56:00,927 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T21:56:00,927 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T21:56:00,927 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T21:56:00,928 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37401-0x1005f228bd00003, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T21:56:00,928 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-20T21:56:00,929 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37401-0x1005f228bd00003, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:56:00,929 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-20T21:56:00,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37401 2024-11-20T21:56:00,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37401 2024-11-20T21:56:00,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37401 2024-11-20T21:56:00,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37401 2024-11-20T21:56:00,931 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37401 2024-11-20T21:56:00,932 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.HRegionServer(746): ClusterId : 7142a613-be64-40b9-806c-ea0e1eb00ed4 2024-11-20T21:56:00,932 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T21:56:00,933 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T21:56:00,934 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T21:56:00,935 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T21:56:00,936 DEBUG [RS:2;e1fae5a3a175:37401 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d071c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e1fae5a3a175/172.17.0.2:0 2024-11-20T21:56:00,936 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-20T21:56:00,936 DEBUG [Thread-201 {}] bucket.BucketCache(1584): Cached File Entry:> 2024-11-20T21:56:00,937 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T21:56:00,937 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-20T21:56:00,937 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.persistence is complete. 2024-11-20T21:56:00,950 DEBUG [RS:2;e1fae5a3a175:37401 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;e1fae5a3a175:37401 2024-11-20T21:56:00,950 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T21:56:00,950 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T21:56:00,950 DEBUG [RS:2;e1fae5a3a175:37401 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T21:56:00,951 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.HRegionServer(2659): reportForDuty to master=e1fae5a3a175,43385,1732139757425 with port=37401, startcode=1732139760919 2024-11-20T21:56:00,951 DEBUG [RS:2;e1fae5a3a175:37401 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T21:56:00,953 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55317, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T21:56:00,953 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e1fae5a3a175,37401,1732139760919 2024-11-20T21:56:00,954 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] master.ServerManager(517): Registering regionserver=e1fae5a3a175,37401,1732139760919 2024-11-20T21:56:00,955 DEBUG [RS:2;e1fae5a3a175:37401 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39 2024-11-20T21:56:00,955 DEBUG [RS:2;e1fae5a3a175:37401 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T21:56:00,955 DEBUG [RS:2;e1fae5a3a175:37401 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T21:56:00,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:56:00,957 DEBUG [RS:2;e1fae5a3a175:37401 {}] zookeeper.ZKUtil(111): regionserver:37401-0x1005f228bd00003, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e1fae5a3a175,37401,1732139760919 2024-11-20T21:56:00,958 WARN [RS:2;e1fae5a3a175:37401 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:56:00,958 INFO [RS:2;e1fae5a3a175:37401 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:56:00,958 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e1fae5a3a175,37401,1732139760919] 2024-11-20T21:56:00,958 DEBUG [RS:2;e1fae5a3a175:37401 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,37401,1732139760919 2024-11-20T21:56:00,962 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:00,962 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 6377002468577638191cf24457a2724e, NAME => 'testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:56:00,963 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732139758712 6377002468577638191cf24457a2724e 2024-11-20T21:56:00,963 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:56:00,963 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 6377002468577638191cf24457a2724e 2024-11-20T21:56:00,963 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 6377002468577638191cf24457a2724e 2024-11-20T21:56:00,964 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T21:56:00,964 INFO [StoreOpener-6377002468577638191cf24457a2724e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 6377002468577638191cf24457a2724e 2024-11-20T21:56:00,965 INFO [StoreOpener-6377002468577638191cf24457a2724e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6377002468577638191cf24457a2724e columnFamilyName family 2024-11-20T21:56:00,966 DEBUG [StoreOpener-6377002468577638191cf24457a2724e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:56:00,966 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T21:56:00,967 DEBUG [StoreFileOpener-6377002468577638191cf24457a2724e-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/family/aff55f404a254870b8fa95d59f134e73, delay=990 ms 2024-11-20T21:56:00,968 INFO [RS:2;e1fae5a3a175:37401 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T21:56:00,968 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:56:00,968 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T21:56:00,968 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T21:56:00,969 DEBUG [StoreOpener-6377002468577638191cf24457a2724e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/family/aff55f404a254870b8fa95d59f134e73 2024-11-20T21:56:00,969 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T21:56:00,969 INFO [StoreOpener-6377002468577638191cf24457a2724e-1 {}] regionserver.HStore(327): Store=6377002468577638191cf24457a2724e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:56:00,969 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:56:00,969 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:56:00,969 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 6377002468577638191cf24457a2724e 2024-11-20T21:56:00,969 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:56:00,969 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:56:00,969 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:56:00,969 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e1fae5a3a175:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:56:00,969 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:56:00,970 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:56:00,970 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:56:00,970 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:56:00,970 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:56:00,970 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e1fae5a3a175:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:56:00,970 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e 2024-11-20T21:56:00,970 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e1fae5a3a175:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:56:00,970 DEBUG [RS:2;e1fae5a3a175:37401 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e1fae5a3a175:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:56:00,970 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e 2024-11-20T21:56:00,971 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 6377002468577638191cf24457a2724e 2024-11-20T21:56:00,971 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 6377002468577638191cf24457a2724e 2024-11-20T21:56:00,972 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 6377002468577638191cf24457a2724e 2024-11-20T21:56:00,972 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 6377002468577638191cf24457a2724e; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71741333, jitterRate=0.06902916729450226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:56:00,972 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6377002468577638191cf24457a2724e 2024-11-20T21:56:00,973 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:56:00,973 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:56:00,973 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:56:00,973 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T21:56:00,973 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 6377002468577638191cf24457a2724e: Running coprocessor pre-open hook at 1732139760963Writing region info on filesystem at 1732139760963Initializing all the Stores at 1732139760964 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732139760964Cleaning up temporary data from old regions at 1732139760971 (+7 ms)Running coprocessor post-open hooks at 1732139760973 (+2 ms)Region opened successfully at 1732139760973 2024-11-20T21:56:00,973 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T21:56:00,973 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,37401,1732139760919-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:56:00,974 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e., pid=9, masterSystemTime=1732139760960 2024-11-20T21:56:00,977 DEBUG [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:00,977 INFO [RS_OPEN_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:00,978 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=6377002468577638191cf24457a2724e, regionState=OPEN, openSeqNum=10, regionLocation=e1fae5a3a175,39423,1732139757490 2024-11-20T21:56:00,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6377002468577638191cf24457a2724e, server=e1fae5a3a175,39423,1732139757490 because future has completed 2024-11-20T21:56:00,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-20T21:56:00,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 6377002468577638191cf24457a2724e, server=e1fae5a3a175,39423,1732139757490 in 179 msec 2024-11-20T21:56:00,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T21:56:00,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732139758712, region=6377002468577638191cf24457a2724e, ASSIGN in 339 msec 2024-11-20T21:56:00,992 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server e1fae5a3a175,38757,1732139757549 after splitting done 2024-11-20T21:56:00,992 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T21:56:00,992 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1fae5a3a175,37401,1732139760919-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:56:00,993 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:56:00,993 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.Replication(171): e1fae5a3a175,37401,1732139760919 started 2024-11-20T21:56:00,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure e1fae5a3a175,38757,1732139757549, splitWal=true, meta=false in 355 msec 2024-11-20T21:56:01,008 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:56:01,008 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.HRegionServer(1482): Serving as e1fae5a3a175,37401,1732139760919, RpcServer on e1fae5a3a175/172.17.0.2:37401, sessionid=0x1005f228bd00003 2024-11-20T21:56:01,008 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T21:56:01,008 DEBUG [RS:2;e1fae5a3a175:37401 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e1fae5a3a175,37401,1732139760919 2024-11-20T21:56:01,008 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1fae5a3a175,37401,1732139760919' 2024-11-20T21:56:01,008 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T21:56:01,009 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T21:56:01,010 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T21:56:01,010 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T21:56:01,010 DEBUG [RS:2;e1fae5a3a175:37401 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e1fae5a3a175,37401,1732139760919 2024-11-20T21:56:01,010 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1fae5a3a175,37401,1732139760919' 2024-11-20T21:56:01,010 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T21:56:01,010 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T21:56:01,011 DEBUG [RS:2;e1fae5a3a175:37401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T21:56:01,011 INFO [RS:2;e1fae5a3a175:37401 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T21:56:01,011 INFO [RS:2;e1fae5a3a175:37401 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T21:56:01,112 INFO [RS:2;e1fae5a3a175:37401 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:56:01,113 INFO [RS:2;e1fae5a3a175:37401 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e1fae5a3a175%2C37401%2C1732139760919, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,37401,1732139760919, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs, maxLogs=55 2024-11-20T21:56:01,130 INFO [RS:2;e1fae5a3a175:37401 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/WALs/e1fae5a3a175,37401,1732139760919/e1fae5a3a175%2C37401%2C1732139760919.1732139761113 2024-11-20T21:56:01,130 DEBUG [RS:2;e1fae5a3a175:37401 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T21:56:01,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T21:56:01,509 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:56:01,509 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:56:01,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:56:01,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:56:01,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:56:01,509 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T21:56:01,510 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1065962717, stopped=false 2024-11-20T21:56:01,510 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e1fae5a3a175,43385,1732139757425 2024-11-20T21:56:01,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T21:56:01,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37401-0x1005f228bd00003, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T21:56:01,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T21:56:01,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:56:01,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:56:01,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37401-0x1005f228bd00003, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:56:01,511 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T21:56:01,511 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:56:01,512 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:56:01,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:56:01,512 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37401-0x1005f228bd00003, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:56:01,512 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:56:01,512 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e1fae5a3a175,39423,1732139757490' ***** 2024-11-20T21:56:01,512 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T21:56:01,512 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:56:01,512 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e1fae5a3a175,37401,1732139760919' ***** 2024-11-20T21:56:01,512 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T21:56:01,512 INFO [RS:0;e1fae5a3a175:39423 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T21:56:01,513 INFO [RS:2;e1fae5a3a175:37401 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T21:56:01,513 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T21:56:01,513 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T21:56:01,513 INFO [RS:2;e1fae5a3a175:37401 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.persistence 2024-11-20T21:56:01,513 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T21:56:01,513 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T21:56:01,513 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T21:56:01,513 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T21:56:01,513 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T21:56:01,513 INFO [RS:0;e1fae5a3a175:39423 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02a6f552-90a5-0666-00e9-83029f9d796b/bucket.persistence 2024-11-20T21:56:01,513 INFO [RS:2;e1fae5a3a175:37401 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T21:56:01,513 INFO [RS:0;e1fae5a3a175:39423 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T21:56:01,513 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T21:56:01,513 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T21:56:01,517 INFO [regionserver/e1fae5a3a175:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:56:01,523 DEBUG [RS:0;e1fae5a3a175:39423 {}] bucket.BucketCache(1487): Thread RS:0;e1fae5a3a175:39423 started persisting bucket cache to file 2024-11-20T21:56:01,523 DEBUG [RS:2;e1fae5a3a175:37401 {}] bucket.BucketCache(1487): Thread RS:2;e1fae5a3a175:37401 started persisting bucket cache to file 2024-11-20T21:56:01,524 DEBUG [RS:2;e1fae5a3a175:37401 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T21:56:01,524 DEBUG [RS:0;e1fae5a3a175:39423 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T21:56:01,524 DEBUG [RS:2;e1fae5a3a175:37401 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-20T21:56:01,524 DEBUG [RS:0;e1fae5a3a175:39423 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T21:56:01,527 DEBUG [RS:0;e1fae5a3a175:39423 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T21:56:01,527 DEBUG [RS:0;e1fae5a3a175:39423 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732139761523 2024-11-20T21:56:01,527 DEBUG [RS:2;e1fae5a3a175:37401 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-20T21:56:01,527 DEBUG [RS:2;e1fae5a3a175:37401 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732139761523 2024-11-20T21:56:01,527 DEBUG [RS:0;e1fae5a3a175:39423 {}] bucket.BucketCache(1509): Thread RS:0;e1fae5a3a175:39423 finished persisting bucket cache to file, renaming 2024-11-20T21:56:01,527 DEBUG [RS:2;e1fae5a3a175:37401 {}] bucket.BucketCache(1509): Thread RS:2;e1fae5a3a175:37401 finished persisting bucket cache to file, renaming 2024-11-20T21:56:01,527 WARN [RS:2;e1fae5a3a175:37401 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-20T21:56:01,527 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T21:56:01,527 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T21:56:01,527 INFO [RS:0;e1fae5a3a175:39423 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T21:56:01,527 INFO [RS:2;e1fae5a3a175:37401 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T21:56:01,527 INFO [RS:0;e1fae5a3a175:39423 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T21:56:01,527 INFO [RS:2;e1fae5a3a175:37401 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T21:56:01,527 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.HRegionServer(959): stopping server e1fae5a3a175,37401,1732139760919 2024-11-20T21:56:01,527 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:56:01,527 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(3091): Received CLOSE for 6377002468577638191cf24457a2724e 2024-11-20T21:56:01,527 INFO [RS:2;e1fae5a3a175:37401 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;e1fae5a3a175:37401. 2024-11-20T21:56:01,527 DEBUG [RS:2;e1fae5a3a175:37401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:56:01,527 DEBUG [RS:2;e1fae5a3a175:37401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:56:01,527 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(959): stopping server e1fae5a3a175,39423,1732139757490 2024-11-20T21:56:01,528 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:56:01,528 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.HRegionServer(976): stopping server e1fae5a3a175,37401,1732139760919; all regions closed. 2024-11-20T21:56:01,528 INFO [RS:0;e1fae5a3a175:39423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e1fae5a3a175:39423. 2024-11-20T21:56:01,528 DEBUG [RS:0;e1fae5a3a175:39423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:56:01,528 DEBUG [RS:0;e1fae5a3a175:39423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:56:01,528 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T21:56:01,528 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T21:56:01,528 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T21:56:01,528 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T21:56:01,528 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T21:56:01,528 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T21:56:01,528 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6377002468577638191cf24457a2724e, disabling compactions & flushes 2024-11-20T21:56:01,528 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:01,528 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:01,528 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T21:56:01,528 DEBUG [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(1325): Online Regions={6377002468577638191cf24457a2724e=testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e., 1588230740=hbase:meta,,1.1588230740} 2024-11-20T21:56:01,528 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. after waiting 0 ms 2024-11-20T21:56:01,528 DEBUG [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6377002468577638191cf24457a2724e 2024-11-20T21:56:01,528 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T21:56:01,528 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:01,528 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T21:56:01,528 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T21:56:01,529 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T21:56:01,529 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T21:56:01,529 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T21:56:01,529 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-20T21:56:01,530 DEBUG [RS:2;e1fae5a3a175:37401 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs 2024-11-20T21:56:01,530 INFO [RS:2;e1fae5a3a175:37401 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e1fae5a3a175%2C37401%2C1732139760919:(num 1732139761113) 2024-11-20T21:56:01,530 DEBUG [RS:2;e1fae5a3a175:37401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:56:01,530 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:56:01,530 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:56:01,531 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.ChoreService(370): Chore service for: regionserver/e1fae5a3a175:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T21:56:01,531 DEBUG [StoreFileCloser-6377002468577638191cf24457a2724e-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/family/aff55f404a254870b8fa95d59f134e73 2024-11-20T21:56:01,531 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T21:56:01,531 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T21:56:01,531 INFO [regionserver/e1fae5a3a175:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:56:01,531 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T21:56:01,531 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:56:01,531 INFO [RS:2;e1fae5a3a175:37401 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37401 2024-11-20T21:56:01,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:56:01,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37401-0x1005f228bd00003, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e1fae5a3a175,37401,1732139760919 2024-11-20T21:56:01,533 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:56:01,533 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e1fae5a3a175,37401,1732139760919] 2024-11-20T21:56:01,534 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e1fae5a3a175,37401,1732139760919 already deleted, retry=false 2024-11-20T21:56:01,535 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e1fae5a3a175,37401,1732139760919 expired; onlineServers=1 2024-11-20T21:56:01,536 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/default/testBlockEvictionOnGracefulStop1732139758712/6377002468577638191cf24457a2724e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T21:56:01,537 INFO [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:01,537 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6377002468577638191cf24457a2724e: Waiting for close lock at 1732139761528Running coprocessor pre-close hooks at 1732139761528Disabling compacts and flushes for region at 1732139761528Disabling writes for close at 1732139761528Writing region close event to WAL at 1732139761531 (+3 ms)Running coprocessor post-close hooks at 1732139761536 (+5 ms)Closed at 1732139761537 (+1 ms) 2024-11-20T21:56:01,537 DEBUG [RS_CLOSE_REGION-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e. 2024-11-20T21:56:01,539 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ce6e22ab532e447fbf4f4168b96402be 2024-11-20T21:56:01,552 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/info/ce6e22ab532e447fbf4f4168b96402be is 209, key is testBlockEvictionOnGracefulStop1732139758712,,1732139758717.6377002468577638191cf24457a2724e./info:regioninfo/1732139760978/Put/seqid=0 2024-11-20T21:56:01,553 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/info/ce6e22ab532e447fbf4f4168b96402be 2024-11-20T21:56:01,560 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1e4a18d5bc2444f2abb51d5c79058bfd 2024-11-20T21:56:01,572 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/ns/1e4a18d5bc2444f2abb51d5c79058bfd is 43, key is default/ns:d/1732139758636/Put/seqid=0 2024-11-20T21:56:01,573 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/ns/1e4a18d5bc2444f2abb51d5c79058bfd 2024-11-20T21:56:01,574 INFO [regionserver/e1fae5a3a175:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:56:01,580 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ca98a68717ee40a2b1492fa5a5ace3b3 2024-11-20T21:56:01,593 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/table/ca98a68717ee40a2b1492fa5a5ace3b3 is 80, key is testBlockEvictionOnGracefulStop1732139758712/table:state/1732139759104/Put/seqid=0 2024-11-20T21:56:01,594 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/table/ca98a68717ee40a2b1492fa5a5ace3b3 2024-11-20T21:56:01,596 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/info/ce6e22ab532e447fbf4f4168b96402be as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/info/ce6e22ab532e447fbf4f4168b96402be 2024-11-20T21:56:01,597 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/info/ce6e22ab532e447fbf4f4168b96402be, delay=937 ms 2024-11-20T21:56:01,598 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/info/ce6e22ab532e447fbf4f4168b96402be, entries=14, sequenceid=13, filesize=7.5 K 2024-11-20T21:56:01,599 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/ns/1e4a18d5bc2444f2abb51d5c79058bfd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/ns/1e4a18d5bc2444f2abb51d5c79058bfd 2024-11-20T21:56:01,600 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/ns/1e4a18d5bc2444f2abb51d5c79058bfd, delay=940 ms 2024-11-20T21:56:01,600 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/ns/1e4a18d5bc2444f2abb51d5c79058bfd, entries=2, sequenceid=13, filesize=5.0 K 2024-11-20T21:56:01,601 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/.tmp/table/ca98a68717ee40a2b1492fa5a5ace3b3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/table/ca98a68717ee40a2b1492fa5a5ace3b3 2024-11-20T21:56:01,602 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/table/ca98a68717ee40a2b1492fa5a5ace3b3, delay=922 ms 2024-11-20T21:56:01,603 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/table/ca98a68717ee40a2b1492fa5a5ace3b3, entries=2, sequenceid=13, filesize=5.3 K 2024-11-20T21:56:01,604 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=13, compaction requested=false 2024-11-20T21:56:01,604 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/info/ce6e22ab532e447fbf4f4168b96402be 2024-11-20T21:56:01,605 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/ns/1e4a18d5bc2444f2abb51d5c79058bfd 2024-11-20T21:56:01,606 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/table/ca98a68717ee40a2b1492fa5a5ace3b3 2024-11-20T21:56:01,610 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-20T21:56:01,610 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T21:56:01,610 INFO [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T21:56:01,610 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732139761528Running coprocessor pre-close hooks at 1732139761528Disabling compacts and flushes for region at 1732139761528Disabling writes for close at 1732139761529 (+1 ms)Obtaining lock to block concurrent updates at 1732139761529Preparing flush snapshotting stores in 1588230740 at 1732139761529Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732139761529Flushing stores of hbase:meta,,1.1588230740 at 1732139761530 (+1 ms)Flushing 1588230740/info: creating writer at 1732139761530Flushing 1588230740/info: appending metadata at 1732139761552 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732139761552Flushing 1588230740/ns: creating writer at 1732139761555 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732139761572 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732139761572Flushing 1588230740/table: creating writer at 1732139761575 (+3 ms)Flushing 1588230740/table: appending metadata at 1732139761593 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732139761593Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4deaa296: reopening flushed file at 1732139761596 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25403efa: reopening flushed file at 1732139761598 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fe2d59a: reopening flushed file at 1732139761601 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=13, compaction requested=false at 1732139761604 (+3 ms)Writing region close event to WAL at 1732139761606 (+2 ms)Running coprocessor post-close hooks at 1732139761610 (+4 ms)Closed at 1732139761610 2024-11-20T21:56:01,610 DEBUG [RS_CLOSE_META-regionserver/e1fae5a3a175:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T21:56:01,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37401-0x1005f228bd00003, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:56:01,634 INFO [RS:2;e1fae5a3a175:37401 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:56:01,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37401-0x1005f228bd00003, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:56:01,634 INFO [RS:2;e1fae5a3a175:37401 {}] regionserver.HRegionServer(1031): Exiting; stopping=e1fae5a3a175,37401,1732139760919; zookeeper connection closed. 2024-11-20T21:56:01,635 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6ae88c53 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6ae88c53 2024-11-20T21:56:01,729 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(976): stopping server e1fae5a3a175,39423,1732139757490; all regions closed. 2024-11-20T21:56:01,734 DEBUG [RS:0;e1fae5a3a175:39423 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs 2024-11-20T21:56:01,735 INFO [RS:0;e1fae5a3a175:39423 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e1fae5a3a175%2C39423%2C1732139757490.meta:.meta(num 1732139758592) 2024-11-20T21:56:01,738 DEBUG [RS:0;e1fae5a3a175:39423 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/oldWALs 2024-11-20T21:56:01,739 INFO [RS:0;e1fae5a3a175:39423 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e1fae5a3a175%2C39423%2C1732139757490:(num 1732139758168) 2024-11-20T21:56:01,739 DEBUG [RS:0;e1fae5a3a175:39423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:56:01,739 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:56:01,739 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:56:01,739 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.ChoreService(370): Chore service for: regionserver/e1fae5a3a175:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T21:56:01,739 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:56:01,739 INFO [regionserver/e1fae5a3a175:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:56:01,740 INFO [RS:0;e1fae5a3a175:39423 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39423 2024-11-20T21:56:01,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e1fae5a3a175,39423,1732139757490 2024-11-20T21:56:01,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:56:01,741 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:56:01,742 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e1fae5a3a175,39423,1732139757490] 2024-11-20T21:56:01,743 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e1fae5a3a175,39423,1732139757490 already deleted, retry=false 2024-11-20T21:56:01,743 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e1fae5a3a175,39423,1732139757490 expired; onlineServers=0 2024-11-20T21:56:01,743 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e1fae5a3a175,43385,1732139757425' ***** 2024-11-20T21:56:01,743 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T21:56:01,743 INFO [M:0;e1fae5a3a175:43385 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:56:01,743 INFO [M:0;e1fae5a3a175:43385 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:56:01,743 DEBUG [M:0;e1fae5a3a175:43385 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T21:56:01,744 DEBUG [M:0;e1fae5a3a175:43385 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T21:56:01,744 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T21:56:01,744 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster-HFileCleaner.large.0-1732139757825 {}] cleaner.HFileCleaner(306): Exit Thread[master/e1fae5a3a175:0:becomeActiveMaster-HFileCleaner.large.0-1732139757825,5,FailOnTimeoutGroup] 2024-11-20T21:56:01,744 DEBUG [master/e1fae5a3a175:0:becomeActiveMaster-HFileCleaner.small.0-1732139757825 {}] cleaner.HFileCleaner(306): Exit Thread[master/e1fae5a3a175:0:becomeActiveMaster-HFileCleaner.small.0-1732139757825,5,FailOnTimeoutGroup] 2024-11-20T21:56:01,744 INFO [M:0;e1fae5a3a175:43385 {}] hbase.ChoreService(370): Chore service for: master/e1fae5a3a175:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T21:56:01,744 INFO [M:0;e1fae5a3a175:43385 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:56:01,744 DEBUG [M:0;e1fae5a3a175:43385 {}] master.HMaster(1795): Stopping service threads 2024-11-20T21:56:01,744 INFO [M:0;e1fae5a3a175:43385 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T21:56:01,744 INFO [M:0;e1fae5a3a175:43385 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T21:56:01,744 INFO [M:0;e1fae5a3a175:43385 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T21:56:01,744 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T21:56:01,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T21:56:01,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:56:01,745 DEBUG [M:0;e1fae5a3a175:43385 {}] zookeeper.ZKUtil(347): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T21:56:01,745 WARN [M:0;e1fae5a3a175:43385 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T21:56:01,745 INFO [M:0;e1fae5a3a175:43385 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/.lastflushedseqids 2024-11-20T21:56:01,750 INFO [M:0;e1fae5a3a175:43385 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T21:56:01,750 INFO [M:0;e1fae5a3a175:43385 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T21:56:01,751 DEBUG [M:0;e1fae5a3a175:43385 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T21:56:01,751 INFO [M:0;e1fae5a3a175:43385 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:56:01,751 DEBUG [M:0;e1fae5a3a175:43385 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:56:01,751 DEBUG [M:0;e1fae5a3a175:43385 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T21:56:01,751 DEBUG [M:0;e1fae5a3a175:43385 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:56:01,751 INFO [M:0;e1fae5a3a175:43385 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-20T21:56:01,759 DEBUG [M:0;e1fae5a3a175:43385 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b18248c27db74a36a2addb1aed371c6d 2024-11-20T21:56:01,774 DEBUG [M:0;e1fae5a3a175:43385 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b18248c27db74a36a2addb1aed371c6d is 82, key is hbase:meta,,1/info:regioninfo/1732139758621/Put/seqid=0 2024-11-20T21:56:01,775 INFO [M:0;e1fae5a3a175:43385 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b18248c27db74a36a2addb1aed371c6d 2024-11-20T21:56:01,782 DEBUG [M:0;e1fae5a3a175:43385 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0ed629d8fa104b699d253d7e53d3b0e5 2024-11-20T21:56:01,795 DEBUG [M:0;e1fae5a3a175:43385 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0ed629d8fa104b699d253d7e53d3b0e5 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732139759109/Put/seqid=0 2024-11-20T21:56:01,796 INFO [M:0;e1fae5a3a175:43385 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0ed629d8fa104b699d253d7e53d3b0e5 2024-11-20T21:56:01,803 DEBUG [M:0;e1fae5a3a175:43385 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 12eaa494c5a04936aa45a6b74397ea0e 2024-11-20T21:56:01,817 DEBUG [M:0;e1fae5a3a175:43385 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/12eaa494c5a04936aa45a6b74397ea0e is 69, key is e1fae5a3a175,37401,1732139760919/rs:state/1732139760954/Put/seqid=0 2024-11-20T21:56:01,817 INFO [M:0;e1fae5a3a175:43385 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/12eaa494c5a04936aa45a6b74397ea0e 2024-11-20T21:56:01,819 INFO [M:0;e1fae5a3a175:43385 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 12eaa494c5a04936aa45a6b74397ea0e 2024-11-20T21:56:01,819 DEBUG [M:0;e1fae5a3a175:43385 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b18248c27db74a36a2addb1aed371c6d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b18248c27db74a36a2addb1aed371c6d 2024-11-20T21:56:01,821 INFO [M:0;e1fae5a3a175:43385 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b18248c27db74a36a2addb1aed371c6d, entries=8, sequenceid=83, filesize=5.5 K 2024-11-20T21:56:01,821 DEBUG [M:0;e1fae5a3a175:43385 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0ed629d8fa104b699d253d7e53d3b0e5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0ed629d8fa104b699d253d7e53d3b0e5 2024-11-20T21:56:01,823 INFO [M:0;e1fae5a3a175:43385 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0ed629d8fa104b699d253d7e53d3b0e5, entries=9, sequenceid=83, filesize=6.4 K 2024-11-20T21:56:01,823 DEBUG [M:0;e1fae5a3a175:43385 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/12eaa494c5a04936aa45a6b74397ea0e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/12eaa494c5a04936aa45a6b74397ea0e 2024-11-20T21:56:01,825 INFO [M:0;e1fae5a3a175:43385 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 12eaa494c5a04936aa45a6b74397ea0e 2024-11-20T21:56:01,825 INFO [M:0;e1fae5a3a175:43385 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fa33e881-f441-3c53-3793-56870bac496a/6262f2b7-5dfe-1ea8-b763-32d247272e39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/12eaa494c5a04936aa45a6b74397ea0e, entries=3, sequenceid=83, filesize=5.3 K 2024-11-20T21:56:01,826 INFO [M:0;e1fae5a3a175:43385 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 75ms, sequenceid=83, compaction requested=false 2024-11-20T21:56:01,827 INFO [M:0;e1fae5a3a175:43385 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:56:01,828 DEBUG [M:0;e1fae5a3a175:43385 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732139761751Disabling compacts and flushes for region at 1732139761751Disabling writes for close at 1732139761751Obtaining lock to block concurrent updates at 1732139761751Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732139761751Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1732139761751Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732139761752 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732139761752Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732139761774 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732139761774Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732139761776 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732139761795 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732139761795Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732139761798 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732139761816 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732139761816Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65b5625f: reopening flushed file at 1732139761819 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77c001b4: reopening flushed file at 1732139761821 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@155736af: reopening flushed file at 1732139761823 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 75ms, sequenceid=83, compaction requested=false at 1732139761826 (+3 ms)Writing region close event to WAL at 1732139761827 (+1 ms)Closed at 1732139761827 2024-11-20T21:56:01,829 INFO [M:0;e1fae5a3a175:43385 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T21:56:01,829 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:56:01,829 INFO [M:0;e1fae5a3a175:43385 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43385 2024-11-20T21:56:01,829 INFO [M:0;e1fae5a3a175:43385 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:56:01,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:56:01,843 INFO [RS:0;e1fae5a3a175:39423 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:56:01,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39423-0x1005f228bd00001, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:56:01,843 INFO [RS:0;e1fae5a3a175:39423 {}] regionserver.HRegionServer(1031): Exiting; stopping=e1fae5a3a175,39423,1732139757490; zookeeper connection closed. 2024-11-20T21:56:01,843 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a529dae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a529dae 2024-11-20T21:56:01,843 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T21:56:01,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:56:01,931 INFO [M:0;e1fae5a3a175:43385 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:56:01,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1005f228bd00000, quorum=127.0.0.1:57332, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:56:01,937 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T21:56:01,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T21:56:01,939 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:56:01,946 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=241 (was 253), ProcessCount=11 (was 11), AvailableMemoryMB=3791 (was 3885)