2024-11-10 01:49:23,810 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-10 01:49:23,828 main DEBUG Took 0.016419 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 01:49:23,828 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 01:49:23,829 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 01:49:23,830 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 01:49:23,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,837 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 01:49:23,849 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,851 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,852 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,852 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,852 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,853 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,854 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,854 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,854 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,855 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,856 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,856 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,856 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,857 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,857 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,857 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,858 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,858 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,859 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,859 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,859 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,860 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,860 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,860 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:49:23,861 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,861 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 01:49:23,863 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:49:23,864 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 01:49:23,866 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 01:49:23,866 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 01:49:23,867 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 01:49:23,868 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 01:49:23,876 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 01:49:23,879 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 01:49:23,880 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 01:49:23,881 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 01:49:23,881 main DEBUG createAppenders(={Console}) 2024-11-10 01:49:23,882 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-10 01:49:23,882 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-10 01:49:23,882 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-10 01:49:23,883 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 01:49:23,883 main DEBUG OutputStream closed 2024-11-10 01:49:23,883 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 01:49:23,883 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 01:49:23,883 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-10 01:49:23,967 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 01:49:23,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 01:49:23,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 01:49:23,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 01:49:23,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 01:49:23,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 01:49:23,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 01:49:23,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 01:49:23,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 01:49:23,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 01:49:23,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 01:49:23,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 01:49:23,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 01:49:23,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 01:49:23,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 01:49:23,977 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 01:49:23,977 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 01:49:23,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 01:49:23,980 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 01:49:23,980 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-10 01:49:23,981 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 01:49:23,981 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-10T01:49:23,995 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-10 01:49:23,998 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 01:49:23,998 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T01:49:24,220 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f 2024-11-10T01:49:24,243 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=8544 2024-11-10T01:49:24,336 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T01:49:24,433 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:24,450 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/cluster_f3944eaa-43ca-9394-3cff-7d8982a40e59, deleteOnExit=true 2024-11-10T01:49:24,547 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/cluster_f3944eaa-43ca-9394-3cff-7d8982a40e59/zookeeper_0, clientPort=62510, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/cluster_f3944eaa-43ca-9394-3cff-7d8982a40e59/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/cluster_f3944eaa-43ca-9394-3cff-7d8982a40e59/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T01:49:24,562 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62510 2024-11-10T01:49:24,563 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:24,650 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b with version=8 2024-11-10T01:49:24,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/hbase-staging 2024-11-10T01:49:24,760 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T01:49:25,047 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2d55a1c3a545:0 server-side Connection retries=45 2024-11-10T01:49:25,059 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:25,061 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:25,069 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T01:49:25,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:25,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T01:49:25,216 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T01:49:25,310 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T01:49:25,350 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T01:49:25,361 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T01:49:25,365 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T01:49:25,405 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12880 (auto-detected) 2024-11-10T01:49:25,407 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T01:49:25,430 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42483 2024-11-10T01:49:25,457 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42483 connecting to ZooKeeper ensemble=127.0.0.1:62510 2024-11-10T01:49:25,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:424830x0, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T01:49:25,495 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42483-0x1018fde6c170000 connected 2024-11-10T01:49:25,521 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:25,522 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:25,530 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T01:49:25,534 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b, hbase.cluster.distributed=false 2024-11-10T01:49:25,558 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T01:49:25,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42483 2024-11-10T01:49:25,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42483 2024-11-10T01:49:25,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42483 2024-11-10T01:49:25,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42483 2024-11-10T01:49:25,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42483 2024-11-10T01:49:25,657 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2d55a1c3a545:0 server-side Connection retries=45 2024-11-10T01:49:25,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:25,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:25,659 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T01:49:25,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:25,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T01:49:25,662 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T01:49:25,665 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T01:49:25,666 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46789 2024-11-10T01:49:25,668 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46789 connecting to ZooKeeper ensemble=127.0.0.1:62510 2024-11-10T01:49:25,669 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:25,669 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:25,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467890x0, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T01:49:25,674 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46789-0x1018fde6c170001 connected 2024-11-10T01:49:25,674 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T01:49:25,681 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T01:49:25,694 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/bucket.cache 2024-11-10T01:49:25,700 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T01:49:25,701 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T01:49:25,703 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T01:49:25,705 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T01:49:25,705 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T01:49:25,710 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T01:49:25,714 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T01:49:25,714 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/bucket.persistence is complete. 2024-11-10T01:49:25,715 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T01:49:25,718 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T01:49:25,723 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T01:49:25,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46789 2024-11-10T01:49:25,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46789 2024-11-10T01:49:25,726 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46789 2024-11-10T01:49:25,727 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46789 2024-11-10T01:49:25,727 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46789 2024-11-10T01:49:25,755 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2d55a1c3a545:0 server-side Connection retries=45 2024-11-10T01:49:25,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:25,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:25,756 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T01:49:25,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:25,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T01:49:25,757 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T01:49:25,758 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T01:49:25,759 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33877 2024-11-10T01:49:25,761 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33877 connecting to ZooKeeper ensemble=127.0.0.1:62510 2024-11-10T01:49:25,762 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:25,762 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:25,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338770x0, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T01:49:25,768 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33877-0x1018fde6c170002 connected 2024-11-10T01:49:25,768 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T01:49:25,768 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T01:49:25,771 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/bucket.cache 2024-11-10T01:49:25,772 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T01:49:25,772 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T01:49:25,772 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T01:49:25,773 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T01:49:25,773 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T01:49:25,774 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T01:49:25,774 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T01:49:25,774 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/bucket.persistence is complete. 2024-11-10T01:49:25,775 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T01:49:25,776 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T01:49:25,778 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T01:49:25,779 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33877 2024-11-10T01:49:25,779 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33877 2024-11-10T01:49:25,784 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33877 2024-11-10T01:49:25,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33877 2024-11-10T01:49:25,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33877 2024-11-10T01:49:25,805 DEBUG [M:0;2d55a1c3a545:42483 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2d55a1c3a545:42483 2024-11-10T01:49:25,807 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2d55a1c3a545,42483,1731203364837 2024-11-10T01:49:25,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T01:49:25,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T01:49:25,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T01:49:25,819 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2d55a1c3a545,42483,1731203364837 2024-11-10T01:49:25,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T01:49:25,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T01:49:25,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:25,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:25,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:25,844 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T01:49:25,848 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2d55a1c3a545,42483,1731203364837 from backup master directory 2024-11-10T01:49:25,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2d55a1c3a545,42483,1731203364837 2024-11-10T01:49:25,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T01:49:25,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T01:49:25,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T01:49:25,853 WARN [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T01:49:25,853 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2d55a1c3a545,42483,1731203364837 2024-11-10T01:49:25,856 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T01:49:25,858 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T01:49:25,866 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/hbase.id] with ID: 0943aa30-23ab-4d21-aadf-14034dfdcbe9 2024-11-10T01:49:25,866 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/.tmp/hbase.id 2024-11-10T01:49:25,872 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/hbase.id] 2024-11-10T01:49:25,913 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:25,918 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T01:49:25,939 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-10T01:49:25,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:25,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:25,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:25,973 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T01:49:25,977 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T01:49:26,027 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T01:49:26,033 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T01:49:26,091 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store 2024-11-10T01:49:26,112 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T01:49:26,115 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:26,116 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T01:49:26,117 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:26,117 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:26,118 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T01:49:26,118 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:26,119 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:26,119 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731203366116Disabling compacts and flushes for region at 1731203366116Disabling writes for close at 1731203366118 (+2 ms)Writing region close event to WAL at 1731203366119 (+1 ms)Closed at 1731203366119 2024-11-10T01:49:26,120 WARN [master/2d55a1c3a545:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/.initializing 2024-11-10T01:49:26,120 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/WALs/2d55a1c3a545,42483,1731203364837 2024-11-10T01:49:26,134 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T01:49:26,146 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2d55a1c3a545%2C42483%2C1731203364837, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/WALs/2d55a1c3a545,42483,1731203364837, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/oldWALs, maxLogs=10 2024-11-10T01:49:26,185 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/WALs/2d55a1c3a545,42483,1731203364837/2d55a1c3a545%2C42483%2C1731203364837.1731203366151 2024-11-10T01:49:26,186 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T01:49:26,186 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T01:49:26,187 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:26,190 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:26,191 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:26,228 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:26,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T01:49:26,263 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:26,265 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:26,266 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:26,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T01:49:26,272 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:26,273 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T01:49:26,273 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:26,279 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T01:49:26,279 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:26,279 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T01:49:26,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:26,285 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T01:49:26,285 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:26,286 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T01:49:26,286 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:26,291 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:26,292 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:26,296 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:26,297 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:26,300 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T01:49:26,302 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:26,310 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T01:49:26,311 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67138414, jitterRate=4.4032931327819824E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T01:49:26,317 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731203366204Initializing all the Stores at 1731203366204Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203366205 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203366206 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203366206Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203366206Cleaning up temporary data from old regions at 1731203366297 (+91 ms)Region opened successfully at 1731203366317 (+20 ms) 2024-11-10T01:49:26,319 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T01:49:26,361 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e3569d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2d55a1c3a545/172.17.0.2:0 2024-11-10T01:49:26,397 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T01:49:26,409 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T01:49:26,409 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T01:49:26,413 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T01:49:26,413 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T01:49:26,419 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-10T01:49:26,419 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T01:49:26,446 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T01:49:26,455 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T01:49:26,458 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T01:49:26,461 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T01:49:26,462 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T01:49:26,464 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T01:49:26,466 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T01:49:26,470 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T01:49:26,472 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T01:49:26,473 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T01:49:26,475 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T01:49:26,494 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T01:49:26,496 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T01:49:26,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T01:49:26,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T01:49:26,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T01:49:26,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:26,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:26,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:26,511 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2d55a1c3a545,42483,1731203364837, sessionid=0x1018fde6c170000, setting cluster-up flag (Was=false) 2024-11-10T01:49:26,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:26,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:26,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:26,533 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T01:49:26,535 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2d55a1c3a545,42483,1731203364837 2024-11-10T01:49:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:26,546 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T01:49:26,548 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2d55a1c3a545,42483,1731203364837 2024-11-10T01:49:26,553 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T01:49:26,592 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(746): ClusterId : 0943aa30-23ab-4d21-aadf-14034dfdcbe9 2024-11-10T01:49:26,592 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(746): ClusterId : 0943aa30-23ab-4d21-aadf-14034dfdcbe9 2024-11-10T01:49:26,596 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T01:49:26,596 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T01:49:26,602 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T01:49:26,602 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T01:49:26,602 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T01:49:26,602 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T01:49:26,608 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T01:49:26,608 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T01:49:26,609 DEBUG [RS:1;2d55a1c3a545:33877 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e94bfc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2d55a1c3a545/172.17.0.2:0 2024-11-10T01:49:26,609 DEBUG [RS:0;2d55a1c3a545:46789 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b512abb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2d55a1c3a545/172.17.0.2:0 2024-11-10T01:49:26,625 DEBUG [RS:1;2d55a1c3a545:33877 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2d55a1c3a545:33877 2024-11-10T01:49:26,625 DEBUG [RS:0;2d55a1c3a545:46789 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2d55a1c3a545:46789 2024-11-10T01:49:26,628 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T01:49:26,628 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T01:49:26,629 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T01:49:26,629 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T01:49:26,629 DEBUG [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T01:49:26,629 DEBUG [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T01:49:26,631 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T01:49:26,632 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(2659): reportForDuty to master=2d55a1c3a545,42483,1731203364837 with port=46789, startcode=1731203365616 2024-11-10T01:49:26,632 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(2659): reportForDuty to master=2d55a1c3a545,42483,1731203364837 with port=33877, startcode=1731203365754 2024-11-10T01:49:26,646 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T01:49:26,648 DEBUG [RS:1;2d55a1c3a545:33877 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T01:49:26,648 DEBUG [RS:0;2d55a1c3a545:46789 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T01:49:26,653 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T01:49:26,660 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2d55a1c3a545,42483,1731203364837 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T01:49:26,668 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2d55a1c3a545:0, corePoolSize=5, maxPoolSize=5 2024-11-10T01:49:26,668 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2d55a1c3a545:0, corePoolSize=5, maxPoolSize=5 2024-11-10T01:49:26,669 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2d55a1c3a545:0, corePoolSize=5, maxPoolSize=5 2024-11-10T01:49:26,669 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2d55a1c3a545:0, corePoolSize=5, maxPoolSize=5 2024-11-10T01:49:26,669 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2d55a1c3a545:0, corePoolSize=10, maxPoolSize=10 2024-11-10T01:49:26,670 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,670 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2d55a1c3a545:0, corePoolSize=2, maxPoolSize=2 2024-11-10T01:49:26,670 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,672 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731203396672 2024-11-10T01:49:26,674 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T01:49:26,675 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T01:49:26,676 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T01:49:26,676 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T01:49:26,680 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T01:49:26,680 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T01:49:26,681 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T01:49:26,681 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T01:49:26,681 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:26,682 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T01:49:26,688 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,691 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T01:49:26,693 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T01:49:26,693 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T01:49:26,697 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T01:49:26,697 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T01:49:26,702 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2d55a1c3a545:0:becomeActiveMaster-HFileCleaner.large.0-1731203366699,5,FailOnTimeoutGroup] 2024-11-10T01:49:26,708 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2d55a1c3a545:0:becomeActiveMaster-HFileCleaner.small.0-1731203366702,5,FailOnTimeoutGroup] 2024-11-10T01:49:26,708 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,708 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T01:49:26,710 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,711 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,711 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T01:49:26,712 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b 2024-11-10T01:49:26,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:26,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T01:49:26,729 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39149, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T01:49:26,729 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44689, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T01:49:26,733 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T01:49:26,733 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:26,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:26,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T01:49:26,738 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42483 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:26,741 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T01:49:26,741 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:26,741 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42483 {}] master.ServerManager(517): Registering regionserver=2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:26,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:26,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T01:49:26,747 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T01:49:26,747 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:26,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:26,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T01:49:26,752 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T01:49:26,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:26,752 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:26,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T01:49:26,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740 2024-11-10T01:49:26,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740 2024-11-10T01:49:26,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T01:49:26,754 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42483 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2d55a1c3a545,46789,1731203365616 2024-11-10T01:49:26,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T01:49:26,754 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42483 {}] master.ServerManager(517): Registering regionserver=2d55a1c3a545,46789,1731203365616 2024-11-10T01:49:26,755 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T01:49:26,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T01:49:26,759 DEBUG [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b 2024-11-10T01:49:26,759 DEBUG [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b 2024-11-10T01:49:26,759 DEBUG [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T01:49:26,759 DEBUG [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T01:49:26,759 DEBUG [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T01:49:26,759 DEBUG [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T01:49:26,764 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T01:49:26,764 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72243102, jitterRate=0.076506108045578}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T01:49:26,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T01:49:26,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731203366728Initializing all the Stores at 1731203366728Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203366728Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203366728Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203366729 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203366729Cleaning up temporary data from old regions at 1731203366754 (+25 ms)Region opened successfully at 1731203366767 (+13 ms) 2024-11-10T01:49:26,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T01:49:26,768 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T01:49:26,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T01:49:26,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T01:49:26,768 DEBUG [RS:0;2d55a1c3a545:46789 {}] zookeeper.ZKUtil(111): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2d55a1c3a545,46789,1731203365616 2024-11-10T01:49:26,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T01:49:26,768 WARN [RS:0;2d55a1c3a545:46789 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T01:49:26,769 INFO [RS:0;2d55a1c3a545:46789 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T01:49:26,769 DEBUG [RS:1;2d55a1c3a545:33877 {}] zookeeper.ZKUtil(111): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:26,769 DEBUG [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,46789,1731203365616 2024-11-10T01:49:26,769 WARN [RS:1;2d55a1c3a545:33877 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T01:49:26,769 INFO [RS:1;2d55a1c3a545:33877 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T01:49:26,769 DEBUG [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:26,771 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T01:49:26,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731203366768Disabling compacts and flushes for region at 1731203366768Disabling writes for close at 1731203366768Writing region close event to WAL at 1731203366771 (+3 ms)Closed at 1731203366771 2024-11-10T01:49:26,772 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2d55a1c3a545,33877,1731203365754] 2024-11-10T01:49:26,772 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2d55a1c3a545,46789,1731203365616] 2024-11-10T01:49:26,774 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T01:49:26,774 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T01:49:26,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T01:49:26,792 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T01:49:26,795 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T01:49:26,801 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T01:49:26,801 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T01:49:26,818 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T01:49:26,818 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T01:49:26,824 INFO [RS:0;2d55a1c3a545:46789 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T01:49:26,824 INFO [RS:1;2d55a1c3a545:33877 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T01:49:26,825 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,825 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,826 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T01:49:26,826 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T01:49:26,833 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T01:49:26,833 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T01:49:26,835 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,835 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,835 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,835 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,836 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,836 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,836 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,836 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,836 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,836 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,837 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,837 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,837 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2d55a1c3a545:0, corePoolSize=2, maxPoolSize=2 2024-11-10T01:49:26,837 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2d55a1c3a545:0, corePoolSize=2, maxPoolSize=2 2024-11-10T01:49:26,837 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,837 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,837 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,837 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,837 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,838 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,838 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,838 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,838 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,838 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,838 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,838 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:26,838 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2d55a1c3a545:0, corePoolSize=3, maxPoolSize=3 2024-11-10T01:49:26,839 DEBUG [RS:0;2d55a1c3a545:46789 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2d55a1c3a545:0, corePoolSize=3, maxPoolSize=3 2024-11-10T01:49:26,839 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2d55a1c3a545:0, corePoolSize=3, maxPoolSize=3 2024-11-10T01:49:26,839 DEBUG [RS:1;2d55a1c3a545:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2d55a1c3a545:0, corePoolSize=3, maxPoolSize=3 2024-11-10T01:49:26,842 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,842 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,842 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,842 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,842 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,842 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,842 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,842 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,843 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,843 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,46789,1731203365616-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T01:49:26,843 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,843 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,33877,1731203365754-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T01:49:26,866 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T01:49:26,868 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,46789,1731203365616-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,868 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,868 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.Replication(171): 2d55a1c3a545,46789,1731203365616 started 2024-11-10T01:49:26,873 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T01:49:26,874 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,33877,1731203365754-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,874 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,874 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.Replication(171): 2d55a1c3a545,33877,1731203365754 started 2024-11-10T01:49:26,890 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,890 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(1482): Serving as 2d55a1c3a545,46789,1731203365616, RpcServer on 2d55a1c3a545/172.17.0.2:46789, sessionid=0x1018fde6c170001 2024-11-10T01:49:26,891 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T01:49:26,891 DEBUG [RS:0;2d55a1c3a545:46789 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2d55a1c3a545,46789,1731203365616 2024-11-10T01:49:26,892 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2d55a1c3a545,46789,1731203365616' 2024-11-10T01:49:26,892 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T01:49:26,893 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T01:49:26,894 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T01:49:26,894 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T01:49:26,894 DEBUG [RS:0;2d55a1c3a545:46789 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2d55a1c3a545,46789,1731203365616 2024-11-10T01:49:26,894 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2d55a1c3a545,46789,1731203365616' 2024-11-10T01:49:26,894 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T01:49:26,895 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T01:49:26,896 DEBUG [RS:0;2d55a1c3a545:46789 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T01:49:26,896 INFO [RS:0;2d55a1c3a545:46789 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T01:49:26,896 INFO [RS:0;2d55a1c3a545:46789 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T01:49:26,902 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:26,902 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(1482): Serving as 2d55a1c3a545,33877,1731203365754, RpcServer on 2d55a1c3a545/172.17.0.2:33877, sessionid=0x1018fde6c170002 2024-11-10T01:49:26,902 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T01:49:26,903 DEBUG [RS:1;2d55a1c3a545:33877 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:26,903 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2d55a1c3a545,33877,1731203365754' 2024-11-10T01:49:26,903 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T01:49:26,904 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T01:49:26,904 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T01:49:26,904 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T01:49:26,905 DEBUG [RS:1;2d55a1c3a545:33877 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:26,905 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2d55a1c3a545,33877,1731203365754' 2024-11-10T01:49:26,905 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T01:49:26,905 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T01:49:26,906 DEBUG [RS:1;2d55a1c3a545:33877 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T01:49:26,906 INFO [RS:1;2d55a1c3a545:33877 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T01:49:26,906 INFO [RS:1;2d55a1c3a545:33877 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T01:49:26,946 WARN [2d55a1c3a545:42483 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T01:49:27,001 INFO [RS:0;2d55a1c3a545:46789 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T01:49:27,003 INFO [RS:0;2d55a1c3a545:46789 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2d55a1c3a545%2C46789%2C1731203365616, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,46789,1731203365616, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs, maxLogs=55 2024-11-10T01:49:27,007 INFO [RS:1;2d55a1c3a545:33877 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T01:49:27,008 INFO [RS:1;2d55a1c3a545:33877 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2d55a1c3a545%2C33877%2C1731203365754, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,33877,1731203365754, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs, maxLogs=55 2024-11-10T01:49:27,024 INFO [RS:0;2d55a1c3a545:46789 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,46789,1731203365616/2d55a1c3a545%2C46789%2C1731203365616.1731203367005 2024-11-10T01:49:27,024 DEBUG [RS:0;2d55a1c3a545:46789 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T01:49:27,028 INFO [RS:1;2d55a1c3a545:33877 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,33877,1731203365754/2d55a1c3a545%2C33877%2C1731203365754.1731203367010 2024-11-10T01:49:27,030 DEBUG [RS:1;2d55a1c3a545:33877 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T01:49:27,199 DEBUG [2d55a1c3a545:42483 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-10T01:49:27,215 DEBUG [2d55a1c3a545:42483 {}] balancer.BalancerClusterState(204): Hosts are {2d55a1c3a545=0} racks are {/default-rack=0} 2024-11-10T01:49:27,218 DEBUG [2d55a1c3a545:42483 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-10T01:49:27,219 DEBUG [2d55a1c3a545:42483 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-10T01:49:27,219 DEBUG [2d55a1c3a545:42483 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-10T01:49:27,219 DEBUG [2d55a1c3a545:42483 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-10T01:49:27,219 INFO [2d55a1c3a545:42483 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-10T01:49:27,219 INFO [2d55a1c3a545:42483 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-10T01:49:27,219 DEBUG [2d55a1c3a545:42483 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-10T01:49:27,226 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:27,232 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2d55a1c3a545,33877,1731203365754, state=OPENING 2024-11-10T01:49:27,237 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T01:49:27,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:27,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:27,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:27,241 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T01:49:27,241 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T01:49:27,241 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T01:49:27,242 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T01:49:27,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2d55a1c3a545,33877,1731203365754}] 2024-11-10T01:49:27,417 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T01:49:27,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47843, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T01:49:27,429 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T01:49:27,429 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T01:49:27,430 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T01:49:27,431 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2d55a1c3a545%2C33877%2C1731203365754.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,33877,1731203365754, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs, maxLogs=55 2024-11-10T01:49:27,452 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,33877,1731203365754/2d55a1c3a545%2C33877%2C1731203365754.meta.1731203367433.meta 2024-11-10T01:49:27,453 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T01:49:27,453 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T01:49:27,455 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T01:49:27,458 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T01:49:27,463 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T01:49:27,467 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T01:49:27,468 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:27,468 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T01:49:27,468 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T01:49:27,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T01:49:27,471 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T01:49:27,471 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:27,471 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:27,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T01:49:27,472 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T01:49:27,473 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:27,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:27,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T01:49:27,474 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T01:49:27,474 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:27,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:27,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T01:49:27,475 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T01:49:27,475 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:27,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:27,475 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T01:49:27,477 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740 2024-11-10T01:49:27,477 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740 2024-11-10T01:49:27,478 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T01:49:27,478 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T01:49:27,478 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T01:49:27,479 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T01:49:27,480 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67490059, jitterRate=0.005680248141288757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T01:49:27,480 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T01:49:27,482 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731203367469Writing region info on filesystem at 1731203367469Initializing all the Stores at 1731203367469Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203367470 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203367470Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203367470Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203367470Cleaning up temporary data from old regions at 1731203367478 (+8 ms)Running coprocessor post-open hooks at 1731203367480 (+2 ms)Region opened successfully at 1731203367482 (+2 ms) 2024-11-10T01:49:27,488 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731203367408 2024-11-10T01:49:27,500 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:27,500 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T01:49:27,501 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T01:49:27,501 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2d55a1c3a545,33877,1731203365754, state=OPEN 2024-11-10T01:49:27,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T01:49:27,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T01:49:27,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T01:49:27,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T01:49:27,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T01:49:27,505 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T01:49:27,505 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:27,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T01:49:27,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2d55a1c3a545,33877,1731203365754 in 262 msec 2024-11-10T01:49:27,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T01:49:27,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 728 msec 2024-11-10T01:49:27,514 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T01:49:27,514 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T01:49:27,536 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T01:49:27,537 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2d55a1c3a545,33877,1731203365754, seqNum=-1] 2024-11-10T01:49:27,558 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T01:49:27,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51105, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T01:49:27,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0010 sec 2024-11-10T01:49:27,584 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731203367584, completionTime=-1 2024-11-10T01:49:27,586 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-10T01:49:27,586 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T01:49:27,612 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-10T01:49:27,612 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731203427612 2024-11-10T01:49:27,613 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731203487612 2024-11-10T01:49:27,613 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-10T01:49:27,615 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-10T01:49:27,621 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,42483,1731203364837-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:27,621 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,42483,1731203364837-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:27,621 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,42483,1731203364837-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:27,623 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2d55a1c3a545:42483, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:27,623 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:27,624 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:27,631 DEBUG [master/2d55a1c3a545:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T01:49:27,671 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.818sec 2024-11-10T01:49:27,676 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T01:49:27,677 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T01:49:27,678 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T01:49:27,678 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T01:49:27,678 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T01:49:27,679 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,42483,1731203364837-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T01:49:27,680 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,42483,1731203364837-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T01:49:27,681 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T01:49:27,682 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T01:49:27,682 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,42483,1731203364837-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:27,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218aa893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T01:49:27,704 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T01:49:27,704 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T01:49:27,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2d55a1c3a545,42483,-1 for getting cluster id 2024-11-10T01:49:27,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T01:49:27,719 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0943aa30-23ab-4d21-aadf-14034dfdcbe9' 2024-11-10T01:49:27,722 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T01:49:27,722 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0943aa30-23ab-4d21-aadf-14034dfdcbe9" 2024-11-10T01:49:27,722 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d91ae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T01:49:27,722 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2d55a1c3a545,42483,-1] 2024-11-10T01:49:27,725 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T01:49:27,727 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:27,728 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T01:49:27,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f82a59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T01:49:27,732 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T01:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2d55a1c3a545,33877,1731203365754, seqNum=-1] 2024-11-10T01:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T01:49:27,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T01:49:27,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2d55a1c3a545,42483,1731203364837 2024-11-10T01:49:27,765 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T01:49:27,770 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2d55a1c3a545,42483,1731203364837 2024-11-10T01:49:27,773 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6efe4bc3 2024-11-10T01:49:27,774 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T01:49:27,776 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T01:49:27,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42483 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731203367762', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T01:49:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42483 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731203367762 2024-11-10T01:49:27,790 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731203367762 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T01:49:27,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42483 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731203367762" procId is: 4 2024-11-10T01:49:27,793 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:27,794 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731203367762 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T01:49:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T01:49:27,810 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731203367762-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c11e60cbdce760c591cf9ab4a662e142, NAME => 'testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731203367762', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b 2024-11-10T01:49:27,818 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731203367762-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:27,818 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731203367762-pool-0 {}] regionserver.HRegion(1722): Closing c11e60cbdce760c591cf9ab4a662e142, disabling compactions & flushes 2024-11-10T01:49:27,818 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731203367762-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:27,818 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731203367762-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:27,818 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731203367762-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. after waiting 0 ms 2024-11-10T01:49:27,818 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731203367762-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:27,818 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731203367762-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:27,818 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731203367762-pool-0 {}] regionserver.HRegion(1676): Region close journal for c11e60cbdce760c591cf9ab4a662e142: Waiting for close lock at 1731203367818Disabling compacts and flushes for region at 1731203367818Disabling writes for close at 1731203367818Writing region close event to WAL at 1731203367818Closed at 1731203367818 2024-11-10T01:49:27,820 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731203367762 execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T01:49:27,825 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731203367820"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731203367820"}]},"ts":"1731203367820"} 2024-11-10T01:49:27,829 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T01:49:27,831 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731203367762 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T01:49:27,833 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731203367762","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731203367831"}]},"ts":"1731203367831"} 2024-11-10T01:49:27,837 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731203367762, state=ENABLING in hbase:meta 2024-11-10T01:49:27,838 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2d55a1c3a545=0} racks are {/default-rack=0} 2024-11-10T01:49:27,838 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-10T01:49:27,838 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-10T01:49:27,838 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-10T01:49:27,838 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-10T01:49:27,838 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-10T01:49:27,839 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-10T01:49:27,839 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-10T01:49:27,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731203367762, region=c11e60cbdce760c591cf9ab4a662e142, ASSIGN}] 2024-11-10T01:49:27,842 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731203367762, region=c11e60cbdce760c591cf9ab4a662e142, ASSIGN 2024-11-10T01:49:27,842 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731203367762, region=c11e60cbdce760c591cf9ab4a662e142, ASSIGN; state=OFFLINE, location=2d55a1c3a545,33877,1731203365754; forceNewPlan=false, retain=false 2024-11-10T01:49:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T01:49:27,995 INFO [2d55a1c3a545:42483 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-10T01:49:27,996 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c11e60cbdce760c591cf9ab4a662e142, regionState=OPENING, regionLocation=2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:28,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731203367762, region=c11e60cbdce760c591cf9ab4a662e142, ASSIGN because future has completed 2024-11-10T01:49:28,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c11e60cbdce760c591cf9ab4a662e142, server=2d55a1c3a545,33877,1731203365754}] 2024-11-10T01:49:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T01:49:28,157 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:28,157 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c11e60cbdce760c591cf9ab4a662e142, NAME => 'testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142.', STARTKEY => '', ENDKEY => ''} 2024-11-10T01:49:28,158 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731203367762 c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:28,158 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:28,158 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:28,158 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:28,159 INFO [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:28,163 INFO [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c11e60cbdce760c591cf9ab4a662e142 columnFamilyName family 2024-11-10T01:49:28,164 DEBUG [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:28,164 INFO [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] regionserver.HStore(327): Store=c11e60cbdce760c591cf9ab4a662e142/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T01:49:28,164 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:28,165 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:28,165 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:28,165 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:28,166 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:28,166 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:28,173 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T01:49:28,174 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c11e60cbdce760c591cf9ab4a662e142; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73031865, jitterRate=0.08825959265232086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T01:49:28,174 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:28,175 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c11e60cbdce760c591cf9ab4a662e142: Running coprocessor pre-open hook at 1731203368158Writing region info on filesystem at 1731203368158Initializing all the Stores at 1731203368159 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203368159Cleaning up temporary data from old regions at 1731203368166 (+7 ms)Running coprocessor post-open hooks at 1731203368174 (+8 ms)Region opened successfully at 1731203368175 (+1 ms) 2024-11-10T01:49:28,176 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142., pid=6, masterSystemTime=1731203368154 2024-11-10T01:49:28,179 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:28,179 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:28,180 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c11e60cbdce760c591cf9ab4a662e142, regionState=OPEN, openSeqNum=2, regionLocation=2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:28,183 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c11e60cbdce760c591cf9ab4a662e142, server=2d55a1c3a545,33877,1731203365754 because future has completed 2024-11-10T01:49:28,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T01:49:28,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c11e60cbdce760c591cf9ab4a662e142, server=2d55a1c3a545,33877,1731203365754 in 183 msec 2024-11-10T01:49:28,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T01:49:28,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731203367762, region=c11e60cbdce760c591cf9ab4a662e142, ASSIGN in 346 msec 2024-11-10T01:49:28,189 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731203367762 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T01:49:28,190 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731203367762","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731203368189"}]},"ts":"1731203368189"} 2024-11-10T01:49:28,192 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731203367762, state=ENABLED in hbase:meta 2024-11-10T01:49:28,193 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731203367762 execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T01:49:28,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731203367762 in 409 msec 2024-11-10T01:49:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T01:49:28,429 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731203367762 completed 2024-11-10T01:49:28,429 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731203367762 get assigned. Timeout = 60000ms 2024-11-10T01:49:28,432 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:49:28,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731203367762 assigned to meta. Checking AM states. 2024-11-10T01:49:28,438 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:49:28,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731203367762 assigned. 2024-11-10T01:49:28,447 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731203367762', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142., hostname=2d55a1c3a545,33877,1731203365754, seqNum=2] 2024-11-10T01:49:28,458 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c11e60cbdce760c591cf9ab4a662e142 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-10T01:49:28,483 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 641460dd9da44276b699d140772e7c0c 2024-11-10T01:49:28,521 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/.tmp/family/641460dd9da44276b699d140772e7c0c is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-10T01:49:28,527 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/.tmp/family/641460dd9da44276b699d140772e7c0c 2024-11-10T01:49:28,563 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/.tmp/family/641460dd9da44276b699d140772e7c0c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/family/641460dd9da44276b699d140772e7c0c 2024-11-10T01:49:28,564 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/family/641460dd9da44276b699d140772e7c0c, delay=946 ms 2024-11-10T01:49:28,566 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/family/641460dd9da44276b699d140772e7c0c, entries=2, sequenceid=6, filesize=4.9 K 2024-11-10T01:49:28,571 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for c11e60cbdce760c591cf9ab4a662e142 in 113ms, sequenceid=6, compaction requested=false 2024-11-10T01:49:28,572 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731203367762' 2024-11-10T01:49:28,574 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c11e60cbdce760c591cf9ab4a662e142: 2024-11-10T01:49:29,513 DEBUG [hfile-prefetch-1731203368564 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 641460dd9da44276b699d140772e7c0c_0. 2024-11-10T01:49:29,517 DEBUG [hfile-prefetch-1731203368564 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 641460dd9da44276b699d140772e7c0c_117. 2024-11-10T01:49:29,518 DEBUG [hfile-prefetch-1731203368564 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/family/641460dd9da44276b699d140772e7c0c, with total blocks 2, and data blocks 1 2024-11-10T01:49:29,518 DEBUG [hfile-prefetch-1731203368564 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-10T01:49:29,519 DEBUG [hfile-prefetch-1731203368564 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/family/641460dd9da44276b699d140772e7c0c, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-10T01:49:29,519 DEBUG [hfile-prefetch-1731203368564 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/family/641460dd9da44276b699d140772e7c0c. Will sleep 100ms and try the verification again. 2024-11-10T01:49:29,577 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-10T01:49:29,577 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-10T01:49:29,578 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T01:49:29,578 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T01:49:29,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42483 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=c11e60cbdce760c591cf9ab4a662e142, source=2d55a1c3a545,33877,1731203365754, destination=2d55a1c3a545,46789,1731203365616, warming up region on 2d55a1c3a545,46789,1731203365616 2024-11-10T01:49:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42483 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T01:49:29,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42483 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=c11e60cbdce760c591cf9ab4a662e142, source=2d55a1c3a545,33877,1731203365754, destination=2d55a1c3a545,46789,1731203365616, running balancer 2024-11-10T01:49:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42483 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731203367762, region=c11e60cbdce760c591cf9ab4a662e142, REOPEN/MOVE 2024-11-10T01:49:29,589 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731203367762, region=c11e60cbdce760c591cf9ab4a662e142, REOPEN/MOVE 2024-11-10T01:49:29,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44701, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T01:49:29,590 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c11e60cbdce760c591cf9ab4a662e142, regionState=CLOSING, regionLocation=2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:29,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46789 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46789 {}] regionserver.HRegion(7855): Warmup {ENCODED => c11e60cbdce760c591cf9ab4a662e142, NAME => 'testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142.', STARTKEY => '', ENDKEY => ''} 2024-11-10T01:49:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46789 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:29,592 INFO [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:29,593 INFO [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c11e60cbdce760c591cf9ab4a662e142 columnFamilyName family 2024-11-10T01:49:29,593 DEBUG [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:29,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731203367762, region=c11e60cbdce760c591cf9ab4a662e142, REOPEN/MOVE because future has completed 2024-11-10T01:49:29,594 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-10T01:49:29,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure c11e60cbdce760c591cf9ab4a662e142, server=2d55a1c3a545,33877,1731203365754}] 2024-11-10T01:49:29,597 DEBUG [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/family/641460dd9da44276b699d140772e7c0c 2024-11-10T01:49:29,597 INFO [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] regionserver.HStore(327): Store=c11e60cbdce760c591cf9ab4a662e142/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T01:49:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46789 {}] regionserver.HRegion(1722): Closing c11e60cbdce760c591cf9ab4a662e142, disabling compactions & flushes 2024-11-10T01:49:29,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46789 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46789 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46789 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. after waiting 0 ms 2024-11-10T01:49:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46789 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:29,600 DEBUG [StoreFileCloser-c11e60cbdce760c591cf9ab4a662e142-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/family/641460dd9da44276b699d140772e7c0c 2024-11-10T01:49:29,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46789 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46789 {}] regionserver.HRegion(1676): Region close journal for c11e60cbdce760c591cf9ab4a662e142: Waiting for close lock at 1731203369597Disabling compacts and flushes for region at 1731203369597Disabling writes for close at 1731203369597Writing region close event to WAL at 1731203369601 (+4 ms)Closed at 1731203369601 2024-11-10T01:49:29,602 WARN [hfile-prefetch-1731203368564 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/family/641460dd9da44276b699d140772e7c0c, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-10T01:49:29,603 DEBUG [hfile-prefetch-1731203368564 {}] hfile.PrefetchExecutor(121): Prefetch completed for 641460dd9da44276b699d140772e7c0c 2024-11-10T01:49:29,604 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-10T01:49:29,604 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731203369577 2024-11-10T01:49:29,604 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-10T01:49:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42483 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-10T01:49:29,752 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:29,752 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-10T01:49:29,753 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing c11e60cbdce760c591cf9ab4a662e142, disabling compactions & flushes 2024-11-10T01:49:29,753 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:29,753 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:29,753 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. after waiting 0 ms 2024-11-10T01:49:29,753 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:29,755 DEBUG [StoreFileCloser-c11e60cbdce760c591cf9ab4a662e142-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file 641460dd9da44276b699d140772e7c0c, starting offset: 0, end offset: 9223372036854775807 2024-11-10T01:49:29,756 DEBUG [StoreFileCloser-c11e60cbdce760c591cf9ab4a662e142-family-1 {}] bucket.BucketCache(857): removed key 641460dd9da44276b699d140772e7c0c_0 from back map with offset lock 2094080 in the evict process 2024-11-10T01:49:29,761 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T01:49:29,763 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:29,763 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for c11e60cbdce760c591cf9ab4a662e142: Waiting for close lock at 1731203369753Running coprocessor pre-close hooks at 1731203369753Disabling compacts and flushes for region at 1731203369753Disabling writes for close at 1731203369753Writing region close event to WAL at 1731203369757 (+4 ms)Running coprocessor post-close hooks at 1731203369762 (+5 ms)Closed at 1731203369763 (+1 ms) 2024-11-10T01:49:29,763 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding c11e60cbdce760c591cf9ab4a662e142 move to 2d55a1c3a545,46789,1731203365616 record at close sequenceid=6 2024-11-10T01:49:29,766 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:29,767 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c11e60cbdce760c591cf9ab4a662e142, regionState=CLOSED 2024-11-10T01:49:29,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure c11e60cbdce760c591cf9ab4a662e142, server=2d55a1c3a545,33877,1731203365754 because future has completed 2024-11-10T01:49:29,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T01:49:29,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure c11e60cbdce760c591cf9ab4a662e142, server=2d55a1c3a545,33877,1731203365754 in 176 msec 2024-11-10T01:49:29,773 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731203367762, region=c11e60cbdce760c591cf9ab4a662e142, REOPEN/MOVE; state=CLOSED, location=2d55a1c3a545,46789,1731203365616; forceNewPlan=false, retain=false 2024-11-10T01:49:29,804 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-10T01:49:29,804 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-10T01:49:29,804 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T01:49:29,804 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T01:49:29,807 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-10T01:49:29,807 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-10T01:49:29,807 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T01:49:29,807 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T01:49:29,808 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T01:49:29,808 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731203369804 2024-11-10T01:49:29,808 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-10T01:49:29,810 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T01:49:29,810 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731203369807 2024-11-10T01:49:29,810 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-10T01:49:29,923 INFO [2d55a1c3a545:42483 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-10T01:49:29,924 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c11e60cbdce760c591cf9ab4a662e142, regionState=OPENING, regionLocation=2d55a1c3a545,46789,1731203365616 2024-11-10T01:49:29,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731203367762, region=c11e60cbdce760c591cf9ab4a662e142, REOPEN/MOVE because future has completed 2024-11-10T01:49:29,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c11e60cbdce760c591cf9ab4a662e142, server=2d55a1c3a545,46789,1731203365616}] 2024-11-10T01:49:30,082 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:30,083 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => c11e60cbdce760c591cf9ab4a662e142, NAME => 'testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142.', STARTKEY => '', ENDKEY => ''} 2024-11-10T01:49:30,083 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731203367762 c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,083 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:30,083 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,083 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,084 INFO [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,086 INFO [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c11e60cbdce760c591cf9ab4a662e142 columnFamilyName family 2024-11-10T01:49:30,086 DEBUG [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:30,088 DEBUG [StoreFileOpener-c11e60cbdce760c591cf9ab4a662e142-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/family/641460dd9da44276b699d140772e7c0c, delay=957 ms 2024-11-10T01:49:30,089 DEBUG [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/family/641460dd9da44276b699d140772e7c0c 2024-11-10T01:49:30,089 INFO [StoreOpener-c11e60cbdce760c591cf9ab4a662e142-1 {}] regionserver.HStore(327): Store=c11e60cbdce760c591cf9ab4a662e142/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T01:49:30,089 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,091 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,091 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,092 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,092 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,097 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,098 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened c11e60cbdce760c591cf9ab4a662e142; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74381223, jitterRate=0.10836659371852875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T01:49:30,098 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,099 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for c11e60cbdce760c591cf9ab4a662e142: Running coprocessor pre-open hook at 1731203370084Writing region info on filesystem at 1731203370084Initializing all the Stores at 1731203370084Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203370084Cleaning up temporary data from old regions at 1731203370092 (+8 ms)Running coprocessor post-open hooks at 1731203370098 (+6 ms)Region opened successfully at 1731203370099 (+1 ms) 2024-11-10T01:49:30,100 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142., pid=9, masterSystemTime=1731203370080 2024-11-10T01:49:30,103 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:30,103 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:30,103 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c11e60cbdce760c591cf9ab4a662e142, regionState=OPEN, openSeqNum=10, regionLocation=2d55a1c3a545,46789,1731203365616 2024-11-10T01:49:30,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c11e60cbdce760c591cf9ab4a662e142, server=2d55a1c3a545,46789,1731203365616 because future has completed 2024-11-10T01:49:30,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-10T01:49:30,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure c11e60cbdce760c591cf9ab4a662e142, server=2d55a1c3a545,46789,1731203365616 in 180 msec 2024-11-10T01:49:30,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731203367762, region=c11e60cbdce760c591cf9ab4a662e142, REOPEN/MOVE in 522 msec 2024-11-10T01:49:30,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T01:49:30,141 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T01:49:30,141 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T01:49:30,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:30,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:30,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T01:49:30,147 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T01:49:30,147 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-11-10T01:49:30,147 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2d55a1c3a545,42483,1731203364837 2024-11-10T01:49:30,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T01:49:30,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T01:49:30,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T01:49:30,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,150 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T01:49:30,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,150 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T01:49:30,150 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T01:49:30,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:30,151 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2d55a1c3a545,46789,1731203365616' ***** 2024-11-10T01:49:30,151 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T01:49:30,151 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2d55a1c3a545,33877,1731203365754' ***** 2024-11-10T01:49:30,151 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T01:49:30,151 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T01:49:30,151 INFO [RS:0;2d55a1c3a545:46789 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T01:49:30,151 INFO [RS:1;2d55a1c3a545:33877 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T01:49:30,151 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T01:49:30,152 INFO [RS:1;2d55a1c3a545:33877 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/bucket.persistence 2024-11-10T01:49:30,152 INFO [RS:0;2d55a1c3a545:46789 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/bucket.persistence 2024-11-10T01:49:30,152 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T01:49:30,152 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T01:49:30,152 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T01:49:30,152 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T01:49:30,152 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T01:49:30,152 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T01:49:30,152 INFO [RS:1;2d55a1c3a545:33877 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T01:49:30,152 INFO [RS:0;2d55a1c3a545:46789 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T01:49:30,152 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T01:49:30,152 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T01:49:30,153 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T01:49:30,162 DEBUG [RS:1;2d55a1c3a545:33877 {}] bucket.BucketCache(1488): Thread RS:1;2d55a1c3a545:33877 started persisting bucket cache to file 2024-11-10T01:49:30,162 DEBUG [RS:0;2d55a1c3a545:46789 {}] bucket.BucketCache(1488): Thread RS:0;2d55a1c3a545:46789 started persisting bucket cache to file 2024-11-10T01:49:30,162 DEBUG [RS:1;2d55a1c3a545:33877 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T01:49:30,162 DEBUG [RS:0;2d55a1c3a545:46789 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T01:49:30,162 DEBUG [RS:1;2d55a1c3a545:33877 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T01:49:30,163 DEBUG [RS:0;2d55a1c3a545:46789 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T01:49:30,166 DEBUG [RS:0;2d55a1c3a545:46789 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T01:49:30,166 DEBUG [RS:1;2d55a1c3a545:33877 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T01:49:30,166 DEBUG [RS:0;2d55a1c3a545:46789 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731203370162 2024-11-10T01:49:30,166 DEBUG [RS:1;2d55a1c3a545:33877 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731203370162 2024-11-10T01:49:30,166 DEBUG [RS:1;2d55a1c3a545:33877 {}] bucket.BucketCache(1510): Thread RS:1;2d55a1c3a545:33877 finished persisting bucket cache to file, renaming 2024-11-10T01:49:30,166 DEBUG [RS:0;2d55a1c3a545:46789 {}] bucket.BucketCache(1510): Thread RS:0;2d55a1c3a545:46789 finished persisting bucket cache to file, renaming 2024-11-10T01:49:30,167 WARN [RS:0;2d55a1c3a545:46789 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-10T01:49:30,167 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T01:49:30,167 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T01:49:30,167 INFO [RS:1;2d55a1c3a545:33877 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T01:49:30,167 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T01:49:30,167 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T01:49:30,167 INFO [RS:0;2d55a1c3a545:46789 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T01:49:30,167 INFO [RS:1;2d55a1c3a545:33877 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T01:49:30,167 INFO [RS:0;2d55a1c3a545:46789 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T01:49:30,167 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(959): stopping server 2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:30,167 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T01:49:30,167 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(3091): Received CLOSE for c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,168 INFO [RS:1;2d55a1c3a545:33877 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2d55a1c3a545:33877. 2024-11-10T01:49:30,168 DEBUG [RS:1;2d55a1c3a545:33877 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T01:49:30,168 DEBUG [RS:1;2d55a1c3a545:33877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:30,168 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T01:49:30,168 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T01:49:30,168 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T01:49:30,168 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(959): stopping server 2d55a1c3a545,46789,1731203365616 2024-11-10T01:49:30,168 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T01:49:30,168 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T01:49:30,168 INFO [RS:0;2d55a1c3a545:46789 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2d55a1c3a545:46789. 2024-11-10T01:49:30,168 DEBUG [RS:0;2d55a1c3a545:46789 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T01:49:30,168 DEBUG [RS:0;2d55a1c3a545:46789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:30,169 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T01:49:30,169 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T01:49:30,169 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T01:49:30,169 DEBUG [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T01:49:30,169 DEBUG [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(1325): Online Regions={c11e60cbdce760c591cf9ab4a662e142=testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142.} 2024-11-10T01:49:30,169 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T01:49:30,169 DEBUG [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T01:49:30,169 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c11e60cbdce760c591cf9ab4a662e142, disabling compactions & flushes 2024-11-10T01:49:30,169 DEBUG [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(1351): Waiting on c11e60cbdce760c591cf9ab4a662e142 2024-11-10T01:49:30,169 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T01:49:30,169 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:30,169 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T01:49:30,169 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:30,169 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T01:49:30,169 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. after waiting 0 ms 2024-11-10T01:49:30,169 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T01:49:30,169 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:30,169 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T01:49:30,170 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-10T01:49:30,170 DEBUG [StoreFileCloser-c11e60cbdce760c591cf9ab4a662e142-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/family/641460dd9da44276b699d140772e7c0c 2024-11-10T01:49:30,176 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnRegionMove1731203367762/c11e60cbdce760c591cf9ab4a662e142/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-10T01:49:30,176 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:30,176 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c11e60cbdce760c591cf9ab4a662e142: Waiting for close lock at 1731203370168Running coprocessor pre-close hooks at 1731203370169 (+1 ms)Disabling compacts and flushes for region at 1731203370169Disabling writes for close at 1731203370169Writing region close event to WAL at 1731203370171 (+2 ms)Running coprocessor post-close hooks at 1731203370176 (+5 ms)Closed at 1731203370176 2024-11-10T01:49:30,176 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142. 2024-11-10T01:49:30,180 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1a26d69d51ee430fa3c8dbf34517c272 2024-11-10T01:49:30,200 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/info/1a26d69d51ee430fa3c8dbf34517c272 is 205, key is testBlockEvictionOnRegionMove1731203367762,,1731203367778.c11e60cbdce760c591cf9ab4a662e142./info:regioninfo/1731203370103/Put/seqid=0 2024-11-10T01:49:30,201 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/info/1a26d69d51ee430fa3c8dbf34517c272 2024-11-10T01:49:30,212 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 60f82af64993446aadb297dd873bbdfb 2024-11-10T01:49:30,227 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/ns/60f82af64993446aadb297dd873bbdfb is 43, key is default/ns:d/1731203367567/Put/seqid=0 2024-11-10T01:49:30,228 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/ns/60f82af64993446aadb297dd873bbdfb 2024-11-10T01:49:30,239 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 08e8250590d044f99e11de4d9d964c63 2024-11-10T01:49:30,246 INFO [regionserver/2d55a1c3a545:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T01:49:30,247 INFO [regionserver/2d55a1c3a545:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T01:49:30,254 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/table/08e8250590d044f99e11de4d9d964c63 is 78, key is testBlockEvictionOnRegionMove1731203367762/table:state/1731203368189/Put/seqid=0 2024-11-10T01:49:30,254 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/table/08e8250590d044f99e11de4d9d964c63 2024-11-10T01:49:30,258 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/info/1a26d69d51ee430fa3c8dbf34517c272 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/info/1a26d69d51ee430fa3c8dbf34517c272 2024-11-10T01:49:30,259 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/info/1a26d69d51ee430fa3c8dbf34517c272, delay=945 ms 2024-11-10T01:49:30,261 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/info/1a26d69d51ee430fa3c8dbf34517c272, entries=15, sequenceid=15, filesize=7.6 K 2024-11-10T01:49:30,262 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/ns/60f82af64993446aadb297dd873bbdfb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/ns/60f82af64993446aadb297dd873bbdfb 2024-11-10T01:49:30,263 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/ns/60f82af64993446aadb297dd873bbdfb, delay=908 ms 2024-11-10T01:49:30,265 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/ns/60f82af64993446aadb297dd873bbdfb, entries=2, sequenceid=15, filesize=5.0 K 2024-11-10T01:49:30,266 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/table/08e8250590d044f99e11de4d9d964c63 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/table/08e8250590d044f99e11de4d9d964c63 2024-11-10T01:49:30,267 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/table/08e8250590d044f99e11de4d9d964c63, delay=913 ms 2024-11-10T01:49:30,269 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/table/08e8250590d044f99e11de4d9d964c63, entries=2, sequenceid=15, filesize=5.3 K 2024-11-10T01:49:30,270 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=15, compaction requested=false 2024-11-10T01:49:30,270 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T01:49:30,272 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/info/1a26d69d51ee430fa3c8dbf34517c272 2024-11-10T01:49:30,273 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/ns/60f82af64993446aadb297dd873bbdfb 2024-11-10T01:49:30,273 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/table/08e8250590d044f99e11de4d9d964c63 2024-11-10T01:49:30,278 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-10T01:49:30,279 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T01:49:30,279 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T01:49:30,279 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731203370169Running coprocessor pre-close hooks at 1731203370169Disabling compacts and flushes for region at 1731203370169Disabling writes for close at 1731203370169Obtaining lock to block concurrent updates at 1731203370170 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731203370170Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731203370170Flushing stores of hbase:meta,,1.1588230740 at 1731203370171 (+1 ms)Flushing 1588230740/info: creating writer at 1731203370171Flushing 1588230740/info: appending metadata at 1731203370199 (+28 ms)Flushing 1588230740/info: closing flushed file at 1731203370199Flushing 1588230740/ns: creating writer at 1731203370205 (+6 ms)Flushing 1588230740/ns: appending metadata at 1731203370226 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731203370227 (+1 ms)Flushing 1588230740/table: creating writer at 1731203370232 (+5 ms)Flushing 1588230740/table: appending metadata at 1731203370253 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731203370253Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aceb8f9: reopening flushed file at 1731203370257 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@786623dd: reopening flushed file at 1731203370261 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d0490d4: reopening flushed file at 1731203370265 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=15, compaction requested=false at 1731203370270 (+5 ms)Writing region close event to WAL at 1731203370274 (+4 ms)Running coprocessor post-close hooks at 1731203370278 (+4 ms)Closed at 1731203370279 (+1 ms) 2024-11-10T01:49:30,280 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T01:49:30,369 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(976): stopping server 2d55a1c3a545,33877,1731203365754; all regions closed. 2024-11-10T01:49:30,369 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(976): stopping server 2d55a1c3a545,46789,1731203365616; all regions closed. 2024-11-10T01:49:30,375 DEBUG [RS:1;2d55a1c3a545:33877 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs 2024-11-10T01:49:30,375 DEBUG [RS:0;2d55a1c3a545:46789 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs 2024-11-10T01:49:30,375 INFO [RS:1;2d55a1c3a545:33877 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2d55a1c3a545%2C33877%2C1731203365754.meta:.meta(num 1731203367433) 2024-11-10T01:49:30,375 INFO [RS:0;2d55a1c3a545:46789 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2d55a1c3a545%2C46789%2C1731203365616:(num 1731203367005) 2024-11-10T01:49:30,375 DEBUG [RS:0;2d55a1c3a545:46789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:30,375 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T01:49:30,375 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T01:49:30,376 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.ChoreService(370): Chore service for: regionserver/2d55a1c3a545:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T01:49:30,376 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T01:49:30,376 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T01:49:30,376 INFO [regionserver/2d55a1c3a545:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T01:49:30,376 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T01:49:30,376 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T01:49:30,377 INFO [RS:0;2d55a1c3a545:46789 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46789 2024-11-10T01:49:30,377 DEBUG [RS:1;2d55a1c3a545:33877 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs 2024-11-10T01:49:30,377 INFO [RS:1;2d55a1c3a545:33877 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2d55a1c3a545%2C33877%2C1731203365754:(num 1731203367010) 2024-11-10T01:49:30,377 DEBUG [RS:1;2d55a1c3a545:33877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:30,377 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T01:49:30,377 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T01:49:30,377 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.ChoreService(370): Chore service for: regionserver/2d55a1c3a545:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T01:49:30,378 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T01:49:30,378 INFO [regionserver/2d55a1c3a545:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T01:49:30,378 INFO [RS:1;2d55a1c3a545:33877 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33877 2024-11-10T01:49:30,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2d55a1c3a545,33877,1731203365754 2024-11-10T01:49:30,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T01:49:30,382 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T01:49:30,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2d55a1c3a545,46789,1731203365616 2024-11-10T01:49:30,382 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T01:49:30,384 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2d55a1c3a545,33877,1731203365754] 2024-11-10T01:49:30,386 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2d55a1c3a545,33877,1731203365754 already deleted, retry=false 2024-11-10T01:49:30,386 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2d55a1c3a545,33877,1731203365754 expired; onlineServers=1 2024-11-10T01:49:30,386 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2d55a1c3a545,46789,1731203365616] 2024-11-10T01:49:30,387 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2d55a1c3a545,46789,1731203365616 already deleted, retry=false 2024-11-10T01:49:30,387 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2d55a1c3a545,46789,1731203365616 expired; onlineServers=0 2024-11-10T01:49:30,388 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2d55a1c3a545,42483,1731203364837' ***** 2024-11-10T01:49:30,388 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T01:49:30,388 INFO [M:0;2d55a1c3a545:42483 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T01:49:30,388 INFO [M:0;2d55a1c3a545:42483 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T01:49:30,388 DEBUG [M:0;2d55a1c3a545:42483 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T01:49:30,388 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T01:49:30,388 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster-HFileCleaner.large.0-1731203366699 {}] cleaner.HFileCleaner(306): Exit Thread[master/2d55a1c3a545:0:becomeActiveMaster-HFileCleaner.large.0-1731203366699,5,FailOnTimeoutGroup] 2024-11-10T01:49:30,388 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster-HFileCleaner.small.0-1731203366702 {}] cleaner.HFileCleaner(306): Exit Thread[master/2d55a1c3a545:0:becomeActiveMaster-HFileCleaner.small.0-1731203366702,5,FailOnTimeoutGroup] 2024-11-10T01:49:30,388 DEBUG [M:0;2d55a1c3a545:42483 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T01:49:30,389 INFO [M:0;2d55a1c3a545:42483 {}] hbase.ChoreService(370): Chore service for: master/2d55a1c3a545:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T01:49:30,389 INFO [M:0;2d55a1c3a545:42483 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T01:49:30,389 DEBUG [M:0;2d55a1c3a545:42483 {}] master.HMaster(1795): Stopping service threads 2024-11-10T01:49:30,389 INFO [M:0;2d55a1c3a545:42483 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T01:49:30,389 INFO [M:0;2d55a1c3a545:42483 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T01:49:30,390 INFO [M:0;2d55a1c3a545:42483 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T01:49:30,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T01:49:30,390 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T01:49:30,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,391 DEBUG [M:0;2d55a1c3a545:42483 {}] zookeeper.ZKUtil(347): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T01:49:30,391 WARN [M:0;2d55a1c3a545:42483 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T01:49:30,391 INFO [M:0;2d55a1c3a545:42483 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/.lastflushedseqids 2024-11-10T01:49:30,400 INFO [M:0;2d55a1c3a545:42483 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T01:49:30,400 INFO [M:0;2d55a1c3a545:42483 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T01:49:30,401 DEBUG [M:0;2d55a1c3a545:42483 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T01:49:30,401 INFO [M:0;2d55a1c3a545:42483 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:30,401 DEBUG [M:0;2d55a1c3a545:42483 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:30,401 DEBUG [M:0;2d55a1c3a545:42483 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T01:49:30,401 DEBUG [M:0;2d55a1c3a545:42483 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:30,401 INFO [M:0;2d55a1c3a545:42483 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-10T01:49:30,412 DEBUG [M:0;2d55a1c3a545:42483 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b827a9344d2c4ffba343132fad8c33ed 2024-11-10T01:49:30,426 DEBUG [M:0;2d55a1c3a545:42483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b827a9344d2c4ffba343132fad8c33ed is 82, key is hbase:meta,,1/info:regioninfo/1731203367500/Put/seqid=0 2024-11-10T01:49:30,427 INFO [M:0;2d55a1c3a545:42483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b827a9344d2c4ffba343132fad8c33ed 2024-11-10T01:49:30,436 DEBUG [M:0;2d55a1c3a545:42483 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cea1c831571b483480ac92f642ed69f2 2024-11-10T01:49:30,449 DEBUG [M:0;2d55a1c3a545:42483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cea1c831571b483480ac92f642ed69f2 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731203368195/Put/seqid=0 2024-11-10T01:49:30,450 INFO [M:0;2d55a1c3a545:42483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cea1c831571b483480ac92f642ed69f2 2024-11-10T01:49:30,458 DEBUG [M:0;2d55a1c3a545:42483 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b7eeae6b8bea4ec6931c553133fda7d7 2024-11-10T01:49:30,471 DEBUG [M:0;2d55a1c3a545:42483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b7eeae6b8bea4ec6931c553133fda7d7 is 69, key is 2d55a1c3a545,33877,1731203365754/rs:state/1731203366744/Put/seqid=0 2024-11-10T01:49:30,471 INFO [M:0;2d55a1c3a545:42483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b7eeae6b8bea4ec6931c553133fda7d7 2024-11-10T01:49:30,473 DEBUG [M:0;2d55a1c3a545:42483 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b827a9344d2c4ffba343132fad8c33ed as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b827a9344d2c4ffba343132fad8c33ed 2024-11-10T01:49:30,475 INFO [M:0;2d55a1c3a545:42483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b827a9344d2c4ffba343132fad8c33ed, entries=8, sequenceid=84, filesize=5.5 K 2024-11-10T01:49:30,475 DEBUG [M:0;2d55a1c3a545:42483 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cea1c831571b483480ac92f642ed69f2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cea1c831571b483480ac92f642ed69f2 2024-11-10T01:49:30,477 INFO [M:0;2d55a1c3a545:42483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cea1c831571b483480ac92f642ed69f2, entries=9, sequenceid=84, filesize=6.5 K 2024-11-10T01:49:30,477 DEBUG [M:0;2d55a1c3a545:42483 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b7eeae6b8bea4ec6931c553133fda7d7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b7eeae6b8bea4ec6931c553133fda7d7 2024-11-10T01:49:30,479 INFO [M:0;2d55a1c3a545:42483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b7eeae6b8bea4ec6931c553133fda7d7, entries=2, sequenceid=84, filesize=5.1 K 2024-11-10T01:49:30,479 INFO [M:0;2d55a1c3a545:42483 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=84, compaction requested=false 2024-11-10T01:49:30,482 INFO [M:0;2d55a1c3a545:42483 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:30,482 DEBUG [M:0;2d55a1c3a545:42483 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731203370401Disabling compacts and flushes for region at 1731203370401Disabling writes for close at 1731203370401Obtaining lock to block concurrent updates at 1731203370401Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731203370401Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731203370401Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731203370402 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731203370402Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731203370426 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731203370426Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731203370429 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731203370449 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731203370449Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731203370452 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731203370470 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731203370470Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39dcd4ad: reopening flushed file at 1731203370473 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d2bbdf: reopening flushed file at 1731203370475 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20f1adb4: reopening flushed file at 1731203370477 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=84, compaction requested=false at 1731203370479 (+2 ms)Writing region close event to WAL at 1731203370482 (+3 ms)Closed at 1731203370482 2024-11-10T01:49:30,483 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T01:49:30,483 INFO [M:0;2d55a1c3a545:42483 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T01:49:30,484 INFO [M:0;2d55a1c3a545:42483 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42483 2024-11-10T01:49:30,484 INFO [M:0;2d55a1c3a545:42483 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T01:49:30,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:30,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x1018fde6c170002, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:30,485 INFO [RS:1;2d55a1c3a545:33877 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T01:49:30,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:30,485 INFO [RS:0;2d55a1c3a545:46789 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T01:49:30,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46789-0x1018fde6c170001, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:30,485 INFO [RS:0;2d55a1c3a545:46789 {}] regionserver.HRegionServer(1031): Exiting; stopping=2d55a1c3a545,46789,1731203365616; zookeeper connection closed. 2024-11-10T01:49:30,485 INFO [RS:1;2d55a1c3a545:33877 {}] regionserver.HRegionServer(1031): Exiting; stopping=2d55a1c3a545,33877,1731203365754; zookeeper connection closed. 2024-11-10T01:49:30,486 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@20ee01e8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@20ee01e8 2024-11-10T01:49:30,486 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@18a0ce0d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@18a0ce0d 2024-11-10T01:49:30,486 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-10T01:49:30,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:30,587 INFO [M:0;2d55a1c3a545:42483 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T01:49:30,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42483-0x1018fde6c170000, quorum=127.0.0.1:62510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:30,592 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T01:49:30,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T01:49:30,614 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:30,622 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: master/2d55a1c3a545:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2d55a1c3a545:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731203370260 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731203368564 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731203370088 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/2d55a1c3a545:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2d55a1c3a545:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731203370263 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=151 (was 138) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8204 (was 8544) 2024-11-10T01:49:30,628 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=151, ProcessCount=11, AvailableMemoryMB=8204 2024-11-10T01:49:30,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/hadoop-log-dir so I do NOT create it in target/test-data/13f12625-7381-a6cf-6505-95bcb331db44 2024-11-10T01:49:30,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/hadoop-tmp-dir so I do NOT create it in target/test-data/13f12625-7381-a6cf-6505-95bcb331db44 2024-11-10T01:49:30,629 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:30,632 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/cluster_237826e7-04ab-8c36-950d-daaf1125c90c, deleteOnExit=true 2024-11-10T01:49:30,635 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/cluster_237826e7-04ab-8c36-950d-daaf1125c90c/zookeeper_0, clientPort=53132, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/cluster_237826e7-04ab-8c36-950d-daaf1125c90c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/cluster_237826e7-04ab-8c36-950d-daaf1125c90c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T01:49:30,636 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53132 2024-11-10T01:49:30,648 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b with version=8 2024-11-10T01:49:30,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/hbase-staging 2024-11-10T01:49:30,650 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2d55a1c3a545:0 server-side Connection retries=45 2024-11-10T01:49:30,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:30,651 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:30,651 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T01:49:30,651 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:30,651 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T01:49:30,651 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T01:49:30,651 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T01:49:30,652 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38523 2024-11-10T01:49:30,653 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38523 connecting to ZooKeeper ensemble=127.0.0.1:53132 2024-11-10T01:49:30,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:385230x0, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T01:49:30,659 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38523-0x1018fde83f30000 connected 2024-11-10T01:49:30,674 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:30,674 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:30,675 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T01:49:30,675 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b, hbase.cluster.distributed=false 2024-11-10T01:49:30,677 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T01:49:30,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38523 2024-11-10T01:49:30,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38523 2024-11-10T01:49:30,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38523 2024-11-10T01:49:30,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38523 2024-11-10T01:49:30,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38523 2024-11-10T01:49:30,695 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2d55a1c3a545:0 server-side Connection retries=45 2024-11-10T01:49:30,695 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:30,695 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:30,695 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T01:49:30,695 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:30,695 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T01:49:30,695 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T01:49:30,695 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T01:49:30,696 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36983 2024-11-10T01:49:30,698 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36983 connecting to ZooKeeper ensemble=127.0.0.1:53132 2024-11-10T01:49:30,698 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:30,698 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:30,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:369830x0, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T01:49:30,702 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36983-0x1018fde83f30001 connected 2024-11-10T01:49:30,702 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T01:49:30,702 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T01:49:30,703 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.cache 2024-11-10T01:49:30,703 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T01:49:30,703 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T01:49:30,703 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T01:49:30,704 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T01:49:30,704 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T01:49:30,704 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T01:49:30,704 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T01:49:30,704 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.persistence is complete. 2024-11-10T01:49:30,704 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T01:49:30,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T01:49:30,706 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T01:49:30,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36983 2024-11-10T01:49:30,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36983 2024-11-10T01:49:30,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36983 2024-11-10T01:49:30,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36983 2024-11-10T01:49:30,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36983 2024-11-10T01:49:30,724 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2d55a1c3a545:0 server-side Connection retries=45 2024-11-10T01:49:30,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:30,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:30,724 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T01:49:30,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:30,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T01:49:30,724 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T01:49:30,724 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T01:49:30,725 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34867 2024-11-10T01:49:30,726 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34867 connecting to ZooKeeper ensemble=127.0.0.1:53132 2024-11-10T01:49:30,726 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:30,727 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:30,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348670x0, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T01:49:30,730 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34867-0x1018fde83f30002 connected 2024-11-10T01:49:30,730 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T01:49:30,731 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T01:49:30,733 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.cache 2024-11-10T01:49:30,733 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T01:49:30,733 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T01:49:30,733 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T01:49:30,733 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T01:49:30,734 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T01:49:30,734 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T01:49:30,734 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T01:49:30,734 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T01:49:30,734 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.persistence is complete. 2024-11-10T01:49:30,735 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T01:49:30,736 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T01:49:30,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34867 2024-11-10T01:49:30,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34867 2024-11-10T01:49:30,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34867 2024-11-10T01:49:30,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34867 2024-11-10T01:49:30,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34867 2024-11-10T01:49:30,756 DEBUG [M:0;2d55a1c3a545:38523 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2d55a1c3a545:38523 2024-11-10T01:49:30,757 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2d55a1c3a545,38523,1731203370650 2024-11-10T01:49:30,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T01:49:30,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T01:49:30,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T01:49:30,760 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2d55a1c3a545,38523,1731203370650 2024-11-10T01:49:30,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T01:49:30,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T01:49:30,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,763 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T01:49:30,763 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2d55a1c3a545,38523,1731203370650 from backup master directory 2024-11-10T01:49:30,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2d55a1c3a545,38523,1731203370650 2024-11-10T01:49:30,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T01:49:30,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T01:49:30,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T01:49:30,765 WARN [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T01:49:30,765 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2d55a1c3a545,38523,1731203370650 2024-11-10T01:49:30,766 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/hbase.id] with ID: 4009af69-5961-4217-b28c-93357e2de71e 2024-11-10T01:49:30,766 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/.tmp/hbase.id 2024-11-10T01:49:30,770 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/hbase.id] 2024-11-10T01:49:30,804 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:30,805 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T01:49:30,806 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T01:49:30,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,813 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T01:49:30,814 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T01:49:30,814 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T01:49:30,834 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store 2024-11-10T01:49:30,841 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:30,841 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T01:49:30,841 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:30,841 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:30,841 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T01:49:30,841 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:30,841 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:30,841 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731203370841Disabling compacts and flushes for region at 1731203370841Disabling writes for close at 1731203370841Writing region close event to WAL at 1731203370841Closed at 1731203370841 2024-11-10T01:49:30,841 WARN [master/2d55a1c3a545:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/.initializing 2024-11-10T01:49:30,841 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/WALs/2d55a1c3a545,38523,1731203370650 2024-11-10T01:49:30,847 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T01:49:30,850 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2d55a1c3a545%2C38523%2C1731203370650, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/WALs/2d55a1c3a545,38523,1731203370650, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/oldWALs, maxLogs=10 2024-11-10T01:49:30,868 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/WALs/2d55a1c3a545,38523,1731203370650/2d55a1c3a545%2C38523%2C1731203370650.1731203370851 2024-11-10T01:49:30,869 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T01:49:30,869 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T01:49:30,869 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:30,869 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:30,869 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:30,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:30,875 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T01:49:30,875 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:30,875 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:30,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:30,879 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T01:49:30,879 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:30,879 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T01:49:30,880 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:30,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T01:49:30,884 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:30,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T01:49:30,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:30,888 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T01:49:30,888 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:30,888 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T01:49:30,888 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:30,889 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:30,889 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:30,890 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:30,890 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:30,890 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T01:49:30,891 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T01:49:30,900 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T01:49:30,900 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66313229, jitterRate=-0.01185588538646698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T01:49:30,901 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731203370869Initializing all the Stores at 1731203370870 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203370870Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203370870Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203370870Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203370870Cleaning up temporary data from old regions at 1731203370890 (+20 ms)Region opened successfully at 1731203370901 (+11 ms) 2024-11-10T01:49:30,902 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T01:49:30,906 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d4d24d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2d55a1c3a545/172.17.0.2:0 2024-11-10T01:49:30,907 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T01:49:30,907 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T01:49:30,907 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T01:49:30,907 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T01:49:30,907 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T01:49:30,908 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T01:49:30,908 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T01:49:30,910 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T01:49:30,910 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T01:49:30,912 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T01:49:30,912 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T01:49:30,913 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T01:49:30,914 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T01:49:30,915 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T01:49:30,915 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T01:49:30,917 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T01:49:30,918 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T01:49:30,919 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T01:49:30,921 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T01:49:30,922 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T01:49:30,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T01:49:30,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T01:49:30,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T01:49:30,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,925 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2d55a1c3a545,38523,1731203370650, sessionid=0x1018fde83f30000, setting cluster-up flag (Was=false) 2024-11-10T01:49:30,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,932 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T01:49:30,934 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2d55a1c3a545,38523,1731203370650 2024-11-10T01:49:30,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:30,941 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T01:49:30,942 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2d55a1c3a545,38523,1731203370650 2024-11-10T01:49:30,943 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T01:49:30,945 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T01:49:30,945 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T01:49:30,945 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(746): ClusterId : 4009af69-5961-4217-b28c-93357e2de71e 2024-11-10T01:49:30,946 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T01:49:30,946 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T01:49:30,946 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(746): ClusterId : 4009af69-5961-4217-b28c-93357e2de71e 2024-11-10T01:49:30,946 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T01:49:30,946 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2d55a1c3a545,38523,1731203370650 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T01:49:30,949 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T01:49:30,949 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T01:49:30,949 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2d55a1c3a545:0, corePoolSize=5, maxPoolSize=5 2024-11-10T01:49:30,949 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2d55a1c3a545:0, corePoolSize=5, maxPoolSize=5 2024-11-10T01:49:30,949 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T01:49:30,949 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T01:49:30,949 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2d55a1c3a545:0, corePoolSize=5, maxPoolSize=5 2024-11-10T01:49:30,949 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2d55a1c3a545:0, corePoolSize=5, maxPoolSize=5 2024-11-10T01:49:30,949 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2d55a1c3a545:0, corePoolSize=10, maxPoolSize=10 2024-11-10T01:49:30,949 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:30,950 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2d55a1c3a545:0, corePoolSize=2, maxPoolSize=2 2024-11-10T01:49:30,950 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:30,951 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T01:49:30,952 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T01:49:30,952 DEBUG [RS:1;2d55a1c3a545:34867 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb08a27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2d55a1c3a545/172.17.0.2:0 2024-11-10T01:49:30,953 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T01:49:30,953 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T01:49:30,953 DEBUG [RS:0;2d55a1c3a545:36983 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67b1c852, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2d55a1c3a545/172.17.0.2:0 2024-11-10T01:49:30,954 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:30,954 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T01:49:30,962 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731203400962 2024-11-10T01:49:30,963 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T01:49:30,963 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T01:49:30,963 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T01:49:30,963 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T01:49:30,963 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T01:49:30,963 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T01:49:30,964 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:30,968 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T01:49:30,968 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T01:49:30,968 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T01:49:30,970 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T01:49:30,970 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T01:49:30,971 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2d55a1c3a545:0:becomeActiveMaster-HFileCleaner.large.0-1731203370970,5,FailOnTimeoutGroup] 2024-11-10T01:49:30,971 DEBUG [RS:1;2d55a1c3a545:34867 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2d55a1c3a545:34867 2024-11-10T01:49:30,971 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T01:49:30,971 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T01:49:30,971 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2d55a1c3a545:0:becomeActiveMaster-HFileCleaner.small.0-1731203370971,5,FailOnTimeoutGroup] 2024-11-10T01:49:30,971 DEBUG [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T01:49:30,971 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:30,971 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T01:49:30,971 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:30,972 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:30,973 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(2659): reportForDuty to master=2d55a1c3a545,38523,1731203370650 with port=34867, startcode=1731203370723 2024-11-10T01:49:30,973 DEBUG [RS:1;2d55a1c3a545:34867 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T01:49:30,974 DEBUG [RS:0;2d55a1c3a545:36983 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2d55a1c3a545:36983 2024-11-10T01:49:30,974 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T01:49:30,974 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T01:49:30,974 DEBUG [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T01:49:30,975 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(2659): reportForDuty to master=2d55a1c3a545,38523,1731203370650 with port=36983, startcode=1731203370694 2024-11-10T01:49:30,975 DEBUG [RS:0;2d55a1c3a545:36983 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T01:49:30,977 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53709, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T01:49:30,978 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38523 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:30,978 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38523 {}] master.ServerManager(517): Registering regionserver=2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:30,980 DEBUG [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b 2024-11-10T01:49:30,980 DEBUG [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T01:49:30,980 DEBUG [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T01:49:30,981 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42477, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T01:49:30,981 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T01:49:30,981 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b 2024-11-10T01:49:30,982 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38523 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2d55a1c3a545,36983,1731203370694 2024-11-10T01:49:30,982 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38523 {}] master.ServerManager(517): Registering regionserver=2d55a1c3a545,36983,1731203370694 2024-11-10T01:49:30,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T01:49:30,984 DEBUG [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b 2024-11-10T01:49:30,984 DEBUG [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T01:49:30,984 DEBUG [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T01:49:30,984 DEBUG [RS:1;2d55a1c3a545:34867 {}] zookeeper.ZKUtil(111): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:30,984 WARN [RS:1;2d55a1c3a545:34867 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T01:49:30,984 INFO [RS:1;2d55a1c3a545:34867 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T01:49:30,984 DEBUG [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:30,989 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2d55a1c3a545,36983,1731203370694] 2024-11-10T01:49:30,989 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2d55a1c3a545,34867,1731203370723] 2024-11-10T01:49:30,989 DEBUG [RS:0;2d55a1c3a545:36983 {}] zookeeper.ZKUtil(111): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2d55a1c3a545,36983,1731203370694 2024-11-10T01:49:30,989 WARN [RS:0;2d55a1c3a545:36983 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T01:49:30,989 INFO [RS:0;2d55a1c3a545:36983 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T01:49:30,990 DEBUG [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,36983,1731203370694 2024-11-10T01:49:30,995 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T01:49:30,995 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T01:49:30,997 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:30,997 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T01:49:31,000 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T01:49:31,001 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T01:49:31,001 INFO [RS:1;2d55a1c3a545:34867 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T01:49:31,001 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,002 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T01:49:31,002 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:31,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:31,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T01:49:31,004 INFO [RS:0;2d55a1c3a545:36983 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T01:49:31,004 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,004 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T01:49:31,005 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T01:49:31,005 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,005 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,006 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,006 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,006 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,006 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,006 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2d55a1c3a545:0, corePoolSize=2, maxPoolSize=2 2024-11-10T01:49:31,006 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,006 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,006 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,006 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,006 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,007 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,007 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2d55a1c3a545:0, corePoolSize=3, maxPoolSize=3 2024-11-10T01:49:31,007 DEBUG [RS:1;2d55a1c3a545:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2d55a1c3a545:0, corePoolSize=3, maxPoolSize=3 2024-11-10T01:49:31,008 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T01:49:31,008 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T01:49:31,009 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,009 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T01:49:31,009 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,009 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:31,009 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,009 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:31,009 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,009 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T01:49:31,009 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2d55a1c3a545:0, corePoolSize=2, maxPoolSize=2 2024-11-10T01:49:31,010 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,010 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,010 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,010 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,010 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,011 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:31,011 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2d55a1c3a545:0, corePoolSize=3, maxPoolSize=3 2024-11-10T01:49:31,011 DEBUG [RS:0;2d55a1c3a545:36983 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2d55a1c3a545:0, corePoolSize=3, maxPoolSize=3 2024-11-10T01:49:31,012 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,012 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,012 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,012 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,012 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,012 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,34867,1731203370723-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T01:49:31,020 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,020 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,020 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,021 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,021 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,021 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,36983,1731203370694-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T01:49:31,021 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T01:49:31,021 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:31,021 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:31,021 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T01:49:31,028 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T01:49:31,028 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:31,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:31,029 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T01:49:31,029 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740 2024-11-10T01:49:31,029 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740 2024-11-10T01:49:31,030 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T01:49:31,030 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T01:49:31,030 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T01:49:31,030 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T01:49:31,035 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T01:49:31,036 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,34867,1731203370723-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,036 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,036 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.Replication(171): 2d55a1c3a545,34867,1731203370723 started 2024-11-10T01:49:31,038 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T01:49:31,038 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,36983,1731203370694-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,038 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,038 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.Replication(171): 2d55a1c3a545,36983,1731203370694 started 2024-11-10T01:49:31,038 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T01:49:31,038 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60368228, jitterRate=-0.10044330358505249}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T01:49:31,039 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731203370997Initializing all the Stores at 1731203370997Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203370997Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203370997Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203370997Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203370997Cleaning up temporary data from old regions at 1731203371030 (+33 ms)Region opened successfully at 1731203371039 (+9 ms) 2024-11-10T01:49:31,039 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T01:49:31,039 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T01:49:31,039 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T01:49:31,039 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T01:49:31,039 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T01:49:31,040 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T01:49:31,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731203371039Disabling compacts and flushes for region at 1731203371039Disabling writes for close at 1731203371039Writing region close event to WAL at 1731203371040 (+1 ms)Closed at 1731203371040 2024-11-10T01:49:31,041 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T01:49:31,041 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T01:49:31,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T01:49:31,043 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T01:49:31,044 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T01:49:31,053 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,053 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(1482): Serving as 2d55a1c3a545,34867,1731203370723, RpcServer on 2d55a1c3a545/172.17.0.2:34867, sessionid=0x1018fde83f30002 2024-11-10T01:49:31,053 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T01:49:31,053 DEBUG [RS:1;2d55a1c3a545:34867 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:31,053 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2d55a1c3a545,34867,1731203370723' 2024-11-10T01:49:31,053 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T01:49:31,053 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,053 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(1482): Serving as 2d55a1c3a545,36983,1731203370694, RpcServer on 2d55a1c3a545/172.17.0.2:36983, sessionid=0x1018fde83f30001 2024-11-10T01:49:31,054 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T01:49:31,054 DEBUG [RS:0;2d55a1c3a545:36983 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2d55a1c3a545,36983,1731203370694 2024-11-10T01:49:31,054 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2d55a1c3a545,36983,1731203370694' 2024-11-10T01:49:31,054 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T01:49:31,054 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T01:49:31,054 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T01:49:31,054 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T01:49:31,054 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T01:49:31,054 DEBUG [RS:1;2d55a1c3a545:34867 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:31,055 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2d55a1c3a545,34867,1731203370723' 2024-11-10T01:49:31,055 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T01:49:31,055 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T01:49:31,055 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T01:49:31,055 DEBUG [RS:0;2d55a1c3a545:36983 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2d55a1c3a545,36983,1731203370694 2024-11-10T01:49:31,055 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2d55a1c3a545,36983,1731203370694' 2024-11-10T01:49:31,055 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T01:49:31,055 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T01:49:31,055 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T01:49:31,056 DEBUG [RS:1;2d55a1c3a545:34867 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T01:49:31,056 DEBUG [RS:0;2d55a1c3a545:36983 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T01:49:31,056 INFO [RS:0;2d55a1c3a545:36983 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T01:49:31,056 INFO [RS:1;2d55a1c3a545:34867 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T01:49:31,056 INFO [RS:0;2d55a1c3a545:36983 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T01:49:31,056 INFO [RS:1;2d55a1c3a545:34867 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T01:49:31,157 INFO [RS:0;2d55a1c3a545:36983 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T01:49:31,157 INFO [RS:1;2d55a1c3a545:34867 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T01:49:31,158 INFO [RS:1;2d55a1c3a545:34867 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2d55a1c3a545%2C34867%2C1731203370723, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,34867,1731203370723, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs, maxLogs=55 2024-11-10T01:49:31,158 INFO [RS:0;2d55a1c3a545:36983 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2d55a1c3a545%2C36983%2C1731203370694, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,36983,1731203370694, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs, maxLogs=55 2024-11-10T01:49:31,186 INFO [RS:0;2d55a1c3a545:36983 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,36983,1731203370694/2d55a1c3a545%2C36983%2C1731203370694.1731203371160 2024-11-10T01:49:31,186 INFO [RS:1;2d55a1c3a545:34867 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,34867,1731203370723/2d55a1c3a545%2C34867%2C1731203370723.1731203371160 2024-11-10T01:49:31,186 DEBUG [RS:0;2d55a1c3a545:36983 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T01:49:31,186 DEBUG [RS:1;2d55a1c3a545:34867 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T01:49:31,195 DEBUG [2d55a1c3a545:38523 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-10T01:49:31,195 DEBUG [2d55a1c3a545:38523 {}] balancer.BalancerClusterState(204): Hosts are {2d55a1c3a545=0} racks are {/default-rack=0} 2024-11-10T01:49:31,196 DEBUG [2d55a1c3a545:38523 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-10T01:49:31,196 DEBUG [2d55a1c3a545:38523 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-10T01:49:31,196 DEBUG [2d55a1c3a545:38523 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-10T01:49:31,196 DEBUG [2d55a1c3a545:38523 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-10T01:49:31,196 INFO [2d55a1c3a545:38523 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-10T01:49:31,196 INFO [2d55a1c3a545:38523 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-10T01:49:31,196 DEBUG [2d55a1c3a545:38523 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-10T01:49:31,196 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:31,198 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2d55a1c3a545,34867,1731203370723, state=OPENING 2024-11-10T01:49:31,200 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T01:49:31,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:31,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:31,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:31,202 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T01:49:31,202 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T01:49:31,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2d55a1c3a545,34867,1731203370723}] 2024-11-10T01:49:31,202 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T01:49:31,202 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T01:49:31,355 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T01:49:31,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38867, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T01:49:31,360 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T01:49:31,361 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T01:49:31,361 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T01:49:31,363 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2d55a1c3a545%2C34867%2C1731203370723.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,34867,1731203370723, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs, maxLogs=55 2024-11-10T01:49:31,388 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,34867,1731203370723/2d55a1c3a545%2C34867%2C1731203370723.meta.1731203371363.meta 2024-11-10T01:49:31,389 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T01:49:31,389 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T01:49:31,389 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T01:49:31,389 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T01:49:31,390 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T01:49:31,390 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T01:49:31,390 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:31,390 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T01:49:31,390 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T01:49:31,391 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T01:49:31,392 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T01:49:31,392 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:31,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:31,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T01:49:31,393 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T01:49:31,393 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:31,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:31,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T01:49:31,394 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T01:49:31,394 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:31,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:31,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T01:49:31,395 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T01:49:31,395 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:31,395 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T01:49:31,395 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T01:49:31,396 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740 2024-11-10T01:49:31,396 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740 2024-11-10T01:49:31,396 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T01:49:31,396 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T01:49:31,396 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T01:49:31,398 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T01:49:31,398 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62692730, jitterRate=-0.06580552458763123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T01:49:31,398 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T01:49:31,399 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731203371390Writing region info on filesystem at 1731203371390Initializing all the Stores at 1731203371391 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203371391Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203371391Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203371391Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731203371391Cleaning up temporary data from old regions at 1731203371396 (+5 ms)Running coprocessor post-open hooks at 1731203371398 (+2 ms)Region opened successfully at 1731203371399 (+1 ms) 2024-11-10T01:49:31,400 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731203371355 2024-11-10T01:49:31,402 DEBUG [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T01:49:31,403 INFO [RS_OPEN_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T01:49:31,403 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:31,404 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2d55a1c3a545,34867,1731203370723, state=OPEN 2024-11-10T01:49:31,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T01:49:31,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T01:49:31,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T01:49:31,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T01:49:31,406 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:31,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T01:49:31,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T01:49:31,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T01:49:31,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2d55a1c3a545,34867,1731203370723 in 205 msec 2024-11-10T01:49:31,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T01:49:31,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 367 msec 2024-11-10T01:49:31,412 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T01:49:31,412 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T01:49:31,416 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T01:49:31,416 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2d55a1c3a545,34867,1731203370723, seqNum=-1] 2024-11-10T01:49:31,417 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T01:49:31,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56727, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T01:49:31,424 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 478 msec 2024-11-10T01:49:31,424 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731203371424, completionTime=-1 2024-11-10T01:49:31,424 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-10T01:49:31,424 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T01:49:31,427 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-10T01:49:31,427 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731203431427 2024-11-10T01:49:31,427 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731203491427 2024-11-10T01:49:31,427 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-10T01:49:31,427 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,38523,1731203370650-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,427 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,38523,1731203370650-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,427 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,38523,1731203370650-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,428 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2d55a1c3a545:38523, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,428 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,428 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,430 DEBUG [master/2d55a1c3a545:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T01:49:31,433 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.668sec 2024-11-10T01:49:31,433 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T01:49:31,433 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T01:49:31,433 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T01:49:31,433 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T01:49:31,433 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T01:49:31,433 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,38523,1731203370650-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T01:49:31,433 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,38523,1731203370650-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T01:49:31,434 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T01:49:31,434 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T01:49:31,434 INFO [master/2d55a1c3a545:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,38523,1731203370650-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:31,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T01:49:31,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2d55a1c3a545,38523,-1 for getting cluster id 2024-11-10T01:49:31,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T01:49:31,448 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4009af69-5961-4217-b28c-93357e2de71e' 2024-11-10T01:49:31,449 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T01:49:31,449 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4009af69-5961-4217-b28c-93357e2de71e" 2024-11-10T01:49:31,449 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec23470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T01:49:31,449 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2d55a1c3a545,38523,-1] 2024-11-10T01:49:31,449 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T01:49:31,450 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:31,451 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T01:49:31,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T01:49:31,453 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T01:49:31,454 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2d55a1c3a545,34867,1731203370723, seqNum=-1] 2024-11-10T01:49:31,455 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T01:49:31,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T01:49:31,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2d55a1c3a545,38523,1731203370650 2024-11-10T01:49:31,459 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T01:49:31,460 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 2d55a1c3a545,38523,1731203370650 2024-11-10T01:49:31,460 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61611e0f 2024-11-10T01:49:31,461 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T01:49:31,462 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T01:49:31,463 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731203371459', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T01:49:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731203371459 2024-11-10T01:49:31,466 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731203371459 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T01:49:31,466 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:31,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731203371459" procId is: 4 2024-11-10T01:49:31,467 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731203371459 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T01:49:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T01:49:31,479 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731203371459-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 207d1cdcf3d4218a6da14f099b34a4b8, NAME => 'testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731203371459', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b 2024-11-10T01:49:31,487 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731203371459-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:31,488 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731203371459-pool-0 {}] regionserver.HRegion(1722): Closing 207d1cdcf3d4218a6da14f099b34a4b8, disabling compactions & flushes 2024-11-10T01:49:31,488 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731203371459-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:31,488 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731203371459-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:31,488 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731203371459-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. after waiting 0 ms 2024-11-10T01:49:31,488 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731203371459-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:31,488 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731203371459-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:31,488 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731203371459-pool-0 {}] regionserver.HRegion(1676): Region close journal for 207d1cdcf3d4218a6da14f099b34a4b8: Waiting for close lock at 1731203371488Disabling compacts and flushes for region at 1731203371488Disabling writes for close at 1731203371488Writing region close event to WAL at 1731203371488Closed at 1731203371488 2024-11-10T01:49:31,490 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731203371459 execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T01:49:31,490 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731203371490"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731203371490"}]},"ts":"1731203371490"} 2024-11-10T01:49:31,493 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T01:49:31,494 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731203371459 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T01:49:31,495 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731203371459","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731203371494"}]},"ts":"1731203371494"} 2024-11-10T01:49:31,498 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731203371459, state=ENABLING in hbase:meta 2024-11-10T01:49:31,498 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2d55a1c3a545=0} racks are {/default-rack=0} 2024-11-10T01:49:31,498 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-10T01:49:31,498 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-10T01:49:31,498 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-10T01:49:31,499 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-10T01:49:31,499 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-10T01:49:31,499 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-10T01:49:31,499 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-10T01:49:31,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731203371459, region=207d1cdcf3d4218a6da14f099b34a4b8, ASSIGN}] 2024-11-10T01:49:31,500 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731203371459, region=207d1cdcf3d4218a6da14f099b34a4b8, ASSIGN 2024-11-10T01:49:31,501 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731203371459, region=207d1cdcf3d4218a6da14f099b34a4b8, ASSIGN; state=OFFLINE, location=2d55a1c3a545,36983,1731203370694; forceNewPlan=false, retain=false 2024-11-10T01:49:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T01:49:31,652 INFO [2d55a1c3a545:38523 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-10T01:49:31,652 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=207d1cdcf3d4218a6da14f099b34a4b8, regionState=OPENING, regionLocation=2d55a1c3a545,36983,1731203370694 2024-11-10T01:49:31,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731203371459, region=207d1cdcf3d4218a6da14f099b34a4b8, ASSIGN because future has completed 2024-11-10T01:49:31,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 207d1cdcf3d4218a6da14f099b34a4b8, server=2d55a1c3a545,36983,1731203370694}] 2024-11-10T01:49:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T01:49:31,809 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T01:49:31,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34489, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T01:49:31,813 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:31,814 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 207d1cdcf3d4218a6da14f099b34a4b8, NAME => 'testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8.', STARTKEY => '', ENDKEY => ''} 2024-11-10T01:49:31,814 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731203371459 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:31,814 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:31,814 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:31,814 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:31,815 INFO [StoreOpener-207d1cdcf3d4218a6da14f099b34a4b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:31,820 INFO [StoreOpener-207d1cdcf3d4218a6da14f099b34a4b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 207d1cdcf3d4218a6da14f099b34a4b8 columnFamilyName family 2024-11-10T01:49:31,820 DEBUG [StoreOpener-207d1cdcf3d4218a6da14f099b34a4b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:31,820 INFO [StoreOpener-207d1cdcf3d4218a6da14f099b34a4b8-1 {}] regionserver.HStore(327): Store=207d1cdcf3d4218a6da14f099b34a4b8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T01:49:31,820 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:31,821 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:31,821 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:31,821 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:31,821 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:31,822 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:31,828 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T01:49:31,828 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 207d1cdcf3d4218a6da14f099b34a4b8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67606258, jitterRate=0.007411748170852661}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T01:49:31,829 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:31,830 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 207d1cdcf3d4218a6da14f099b34a4b8: Running coprocessor pre-open hook at 1731203371815Writing region info on filesystem at 1731203371815Initializing all the Stores at 1731203371815Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203371815Cleaning up temporary data from old regions at 1731203371821 (+6 ms)Running coprocessor post-open hooks at 1731203371829 (+8 ms)Region opened successfully at 1731203371830 (+1 ms) 2024-11-10T01:49:31,831 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8., pid=6, masterSystemTime=1731203371809 2024-11-10T01:49:31,833 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:31,833 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:31,834 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=207d1cdcf3d4218a6da14f099b34a4b8, regionState=OPEN, openSeqNum=2, regionLocation=2d55a1c3a545,36983,1731203370694 2024-11-10T01:49:31,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 207d1cdcf3d4218a6da14f099b34a4b8, server=2d55a1c3a545,36983,1731203370694 because future has completed 2024-11-10T01:49:31,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T01:49:31,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 207d1cdcf3d4218a6da14f099b34a4b8, server=2d55a1c3a545,36983,1731203370694 in 181 msec 2024-11-10T01:49:31,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T01:49:31,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731203371459, region=207d1cdcf3d4218a6da14f099b34a4b8, ASSIGN in 340 msec 2024-11-10T01:49:31,842 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731203371459 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T01:49:31,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731203371459","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731203371842"}]},"ts":"1731203371842"} 2024-11-10T01:49:31,845 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731203371459, state=ENABLED in hbase:meta 2024-11-10T01:49:31,846 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731203371459 execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T01:49:31,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731203371459 in 383 msec 2024-11-10T01:49:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T01:49:32,099 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731203371459 completed 2024-11-10T01:49:32,099 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731203371459 get assigned. Timeout = 60000ms 2024-11-10T01:49:32,099 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:49:32,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731203371459 assigned to meta. Checking AM states. 2024-11-10T01:49:32,103 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:49:32,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731203371459 assigned. 2024-11-10T01:49:32,106 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731203371459', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8., hostname=2d55a1c3a545,36983,1731203370694, seqNum=2] 2024-11-10T01:49:32,107 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T01:49:32,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T01:49:32,113 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 207d1cdcf3d4218a6da14f099b34a4b8 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-10T01:49:32,124 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 36a75db902e64d70be28697ff520366d 2024-11-10T01:49:32,138 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/.tmp/family/36a75db902e64d70be28697ff520366d is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-10T01:49:32,139 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/.tmp/family/36a75db902e64d70be28697ff520366d 2024-11-10T01:49:32,143 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/.tmp/family/36a75db902e64d70be28697ff520366d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/family/36a75db902e64d70be28697ff520366d 2024-11-10T01:49:32,144 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/family/36a75db902e64d70be28697ff520366d, delay=979 ms 2024-11-10T01:49:32,146 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/family/36a75db902e64d70be28697ff520366d, entries=2, sequenceid=6, filesize=4.9 K 2024-11-10T01:49:32,147 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 207d1cdcf3d4218a6da14f099b34a4b8 in 34ms, sequenceid=6, compaction requested=false 2024-11-10T01:49:32,147 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731203371459' 2024-11-10T01:49:32,148 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 207d1cdcf3d4218a6da14f099b34a4b8: 2024-11-10T01:49:32,970 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T01:49:33,124 DEBUG [hfile-prefetch-1731203370088 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 36a75db902e64d70be28697ff520366d_0. 2024-11-10T01:49:33,125 DEBUG [hfile-prefetch-1731203370088 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 36a75db902e64d70be28697ff520366d_117. 2024-11-10T01:49:33,126 DEBUG [hfile-prefetch-1731203370088 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/family/36a75db902e64d70be28697ff520366d, with total blocks 2, and data blocks 1 2024-11-10T01:49:33,126 DEBUG [hfile-prefetch-1731203370088 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-10T01:49:33,126 DEBUG [hfile-prefetch-1731203370088 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/family/36a75db902e64d70be28697ff520366d, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-10T01:49:33,126 DEBUG [hfile-prefetch-1731203370088 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/family/36a75db902e64d70be28697ff520366d. Will sleep 100ms and try the verification again. 2024-11-10T01:49:33,135 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-10T01:49:33,135 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-10T01:49:33,135 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T01:49:33,135 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T01:49:33,139 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T01:49:33,139 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731203373135 2024-11-10T01:49:33,139 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-10T01:49:33,148 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;2d55a1c3a545:36983,5,FailOnTimeoutGroup] 2024-11-10T01:49:33,148 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2d55a1c3a545,36983,1731203370694' ***** 2024-11-10T01:49:33,148 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-10T01:49:33,148 INFO [RS:0;2d55a1c3a545:36983 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T01:49:33,149 INFO [RS:0;2d55a1c3a545:36983 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.persistence 2024-11-10T01:49:33,149 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T01:49:33,149 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T01:49:33,149 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T01:49:33,149 INFO [RS:0;2d55a1c3a545:36983 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T01:49:33,149 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T01:49:33,159 DEBUG [RS:0;2d55a1c3a545:36983 {}] bucket.BucketCache(1488): Thread RS:0;2d55a1c3a545:36983 started persisting bucket cache to file 2024-11-10T01:49:33,159 DEBUG [RS:0;2d55a1c3a545:36983 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T01:49:33,159 DEBUG [RS:0;2d55a1c3a545:36983 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T01:49:33,162 DEBUG [RS:0;2d55a1c3a545:36983 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-10T01:49:33,162 DEBUG [RS:0;2d55a1c3a545:36983 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731203373159 2024-11-10T01:49:33,162 DEBUG [RS:0;2d55a1c3a545:36983 {}] bucket.BucketCache(1510): Thread RS:0;2d55a1c3a545:36983 finished persisting bucket cache to file, renaming 2024-11-10T01:49:33,163 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T01:49:33,163 INFO [RS:0;2d55a1c3a545:36983 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T01:49:33,163 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T01:49:33,163 INFO [RS:0;2d55a1c3a545:36983 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T01:49:33,163 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(3091): Received CLOSE for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,163 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(959): stopping server 2d55a1c3a545,36983,1731203370694 2024-11-10T01:49:33,163 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T01:49:33,163 INFO [RS:0;2d55a1c3a545:36983 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2d55a1c3a545:36983. 2024-11-10T01:49:33,163 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T01:49:33,163 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 207d1cdcf3d4218a6da14f099b34a4b8, disabling compactions & flushes 2024-11-10T01:49:33,163 DEBUG [RS:0;2d55a1c3a545:36983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T01:49:33,164 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:33,164 DEBUG [RS:0;2d55a1c3a545:36983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:33,164 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:33,164 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. after waiting 0 ms 2024-11-10T01:49:33,164 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:33,164 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T01:49:33,164 DEBUG [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(1325): Online Regions={207d1cdcf3d4218a6da14f099b34a4b8=testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8.} 2024-11-10T01:49:33,164 DEBUG [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(1351): Waiting on 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,164 DEBUG [StoreFileCloser-207d1cdcf3d4218a6da14f099b34a4b8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/family/36a75db902e64d70be28697ff520366d 2024-11-10T01:49:33,164 WARN [hfile-prefetch-1731203370088 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/family/36a75db902e64d70be28697ff520366d, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-10T01:49:33,165 DEBUG [hfile-prefetch-1731203370088 {}] hfile.PrefetchExecutor(121): Prefetch completed for 36a75db902e64d70be28697ff520366d 2024-11-10T01:49:33,169 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T01:49:33,169 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:33,169 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 207d1cdcf3d4218a6da14f099b34a4b8: Waiting for close lock at 1731203373163Running coprocessor pre-close hooks at 1731203373163Disabling compacts and flushes for region at 1731203373163Disabling writes for close at 1731203373164 (+1 ms)Writing region close event to WAL at 1731203373165 (+1 ms)Running coprocessor post-close hooks at 1731203373169 (+4 ms)Closed at 1731203373169 2024-11-10T01:49:33,170 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:33,223 INFO [regionserver/2d55a1c3a545:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T01:49:33,364 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(976): stopping server 2d55a1c3a545,36983,1731203370694; all regions closed. 2024-11-10T01:49:33,366 DEBUG [RS:0;2d55a1c3a545:36983 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs 2024-11-10T01:49:33,366 INFO [RS:0;2d55a1c3a545:36983 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2d55a1c3a545%2C36983%2C1731203370694:(num 1731203371160) 2024-11-10T01:49:33,366 DEBUG [RS:0;2d55a1c3a545:36983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:33,366 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T01:49:33,366 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T01:49:33,366 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.ChoreService(370): Chore service for: regionserver/2d55a1c3a545:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T01:49:33,367 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T01:49:33,367 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T01:49:33,367 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T01:49:33,367 INFO [regionserver/2d55a1c3a545:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T01:49:33,367 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T01:49:33,367 INFO [RS:0;2d55a1c3a545:36983 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36983 2024-11-10T01:49:33,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2d55a1c3a545,36983,1731203370694 2024-11-10T01:49:33,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T01:49:33,371 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T01:49:33,373 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2d55a1c3a545,36983,1731203370694] 2024-11-10T01:49:33,375 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2d55a1c3a545,36983,1731203370694 already deleted, retry=false 2024-11-10T01:49:33,375 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2d55a1c3a545,36983,1731203370694 on 2d55a1c3a545,38523,1731203370650 2024-11-10T01:49:33,379 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2d55a1c3a545,36983,1731203370694, splitWal=true, meta=false 2024-11-10T01:49:33,381 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 2d55a1c3a545,36983,1731203370694 (carryingMeta=false) 2d55a1c3a545,36983,1731203370694/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1f7f92cd[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-10T01:49:33,381 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2d55a1c3a545,36983,1731203370694, splitWal=true, meta=false 2024-11-10T01:49:33,382 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 2d55a1c3a545,36983,1731203370694 had 1 regions 2024-11-10T01:49:33,383 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2d55a1c3a545,36983,1731203370694, splitWal=true, meta=false, isMeta: false 2024-11-10T01:49:33,384 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,36983,1731203370694-splitting 2024-11-10T01:49:33,384 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,36983,1731203370694-splitting dir is empty, no logs to split. 2024-11-10T01:49:33,384 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2d55a1c3a545,36983,1731203370694 WAL count=0, meta=false 2024-11-10T01:49:33,386 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,36983,1731203370694-splitting dir is empty, no logs to split. 2024-11-10T01:49:33,386 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2d55a1c3a545,36983,1731203370694 WAL count=0, meta=false 2024-11-10T01:49:33,386 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 2d55a1c3a545,36983,1731203370694 WAL splitting is done? wals=0, meta=false 2024-11-10T01:49:33,386 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 2d55a1c3a545,36983,1731203370694 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,36983,1731203370694-splitting does not exist 2024-11-10T01:49:33,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731203371459, region=207d1cdcf3d4218a6da14f099b34a4b8, ASSIGN}] 2024-11-10T01:49:33,389 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731203371459, region=207d1cdcf3d4218a6da14f099b34a4b8, ASSIGN 2024-11-10T01:49:33,389 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731203371459, region=207d1cdcf3d4218a6da14f099b34a4b8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-10T01:49:33,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:33,473 INFO [RS:0;2d55a1c3a545:36983 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T01:49:33,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36983-0x1018fde83f30001, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:33,473 INFO [RS:0;2d55a1c3a545:36983 {}] regionserver.HRegionServer(1031): Exiting; stopping=2d55a1c3a545,36983,1731203370694; zookeeper connection closed. 2024-11-10T01:49:33,474 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6d0834ff {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6d0834ff 2024-11-10T01:49:33,540 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=207d1cdcf3d4218a6da14f099b34a4b8, regionState=OPENING, regionLocation=2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:33,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731203371459, region=207d1cdcf3d4218a6da14f099b34a4b8, ASSIGN because future has completed 2024-11-10T01:49:33,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 207d1cdcf3d4218a6da14f099b34a4b8, server=2d55a1c3a545,34867,1731203370723}] 2024-11-10T01:49:33,665 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2d55a1c3a545:0 server-side Connection retries=45 2024-11-10T01:49:33,665 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:33,665 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:33,665 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T01:49:33,665 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T01:49:33,665 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T01:49:33,665 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T01:49:33,665 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T01:49:33,666 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41571 2024-11-10T01:49:33,667 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41571 connecting to ZooKeeper ensemble=127.0.0.1:53132 2024-11-10T01:49:33,668 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:33,668 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:33,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415710x0, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T01:49:33,671 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41571-0x1018fde83f30003 connected 2024-11-10T01:49:33,672 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41571-0x1018fde83f30003, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-10T01:49:33,672 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-10T01:49:33,672 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T01:49:33,673 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.cache 2024-11-10T01:49:33,673 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T01:49:33,673 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T01:49:33,673 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T01:49:33,673 INFO [Thread-200 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T01:49:33,673 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T01:49:33,674 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T01:49:33,674 INFO [Thread-200 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-10T01:49:33,674 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41571-0x1018fde83f30003, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T01:49:33,675 INFO [Thread-200 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-10T01:49:33,676 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41571-0x1018fde83f30003, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T01:49:33,679 DEBUG [Thread-200 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-10T01:49:33,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41571 2024-11-10T01:49:33,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41571 2024-11-10T01:49:33,679 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T01:49:33,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41571 2024-11-10T01:49:33,679 INFO [Thread-200 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-10T01:49:33,679 INFO [Thread-200 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.persistence is complete. 2024-11-10T01:49:33,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41571 2024-11-10T01:49:33,682 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41571 2024-11-10T01:49:33,685 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.HRegionServer(746): ClusterId : 4009af69-5961-4217-b28c-93357e2de71e 2024-11-10T01:49:33,685 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T01:49:33,687 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T01:49:33,687 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T01:49:33,690 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T01:49:33,690 DEBUG [RS:2;2d55a1c3a545:41571 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dffecf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2d55a1c3a545/172.17.0.2:0 2024-11-10T01:49:33,698 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:33,698 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 207d1cdcf3d4218a6da14f099b34a4b8, NAME => 'testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8.', STARTKEY => '', ENDKEY => ''} 2024-11-10T01:49:33,699 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731203371459 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,699 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T01:49:33,699 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,699 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,700 INFO [StoreOpener-207d1cdcf3d4218a6da14f099b34a4b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,701 INFO [StoreOpener-207d1cdcf3d4218a6da14f099b34a4b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 207d1cdcf3d4218a6da14f099b34a4b8 columnFamilyName family 2024-11-10T01:49:33,701 DEBUG [StoreOpener-207d1cdcf3d4218a6da14f099b34a4b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T01:49:33,703 DEBUG [StoreFileOpener-207d1cdcf3d4218a6da14f099b34a4b8-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/family/36a75db902e64d70be28697ff520366d, delay=981 ms 2024-11-10T01:49:33,703 DEBUG [RS:2;2d55a1c3a545:41571 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2d55a1c3a545:41571 2024-11-10T01:49:33,703 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T01:49:33,703 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T01:49:33,703 DEBUG [RS:2;2d55a1c3a545:41571 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T01:49:33,704 DEBUG [StoreOpener-207d1cdcf3d4218a6da14f099b34a4b8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/family/36a75db902e64d70be28697ff520366d 2024-11-10T01:49:33,704 INFO [StoreOpener-207d1cdcf3d4218a6da14f099b34a4b8-1 {}] regionserver.HStore(327): Store=207d1cdcf3d4218a6da14f099b34a4b8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T01:49:33,704 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,704 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.HRegionServer(2659): reportForDuty to master=2d55a1c3a545,38523,1731203370650 with port=41571, startcode=1731203373664 2024-11-10T01:49:33,704 DEBUG [RS:2;2d55a1c3a545:41571 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T01:49:33,705 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,705 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,705 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,705 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,707 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,707 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58851, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T01:49:33,707 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 207d1cdcf3d4218a6da14f099b34a4b8; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62339867, jitterRate=-0.07106359302997589}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T01:49:33,707 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:33,708 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38523 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2d55a1c3a545,41571,1731203373664 2024-11-10T01:49:33,708 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38523 {}] master.ServerManager(517): Registering regionserver=2d55a1c3a545,41571,1731203373664 2024-11-10T01:49:33,708 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 207d1cdcf3d4218a6da14f099b34a4b8: Running coprocessor pre-open hook at 1731203373699Writing region info on filesystem at 1731203373699Initializing all the Stores at 1731203373699Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731203373699Cleaning up temporary data from old regions at 1731203373705 (+6 ms)Running coprocessor post-open hooks at 1731203373707 (+2 ms)Region opened successfully at 1731203373708 (+1 ms) 2024-11-10T01:49:33,709 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8., pid=9, masterSystemTime=1731203373696 2024-11-10T01:49:33,709 DEBUG [RS:2;2d55a1c3a545:41571 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b 2024-11-10T01:49:33,709 DEBUG [RS:2;2d55a1c3a545:41571 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T01:49:33,709 DEBUG [RS:2;2d55a1c3a545:41571 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T01:49:33,711 DEBUG [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:33,711 INFO [RS_OPEN_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:33,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T01:49:33,711 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=207d1cdcf3d4218a6da14f099b34a4b8, regionState=OPEN, openSeqNum=10, regionLocation=2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:33,712 DEBUG [RS:2;2d55a1c3a545:41571 {}] zookeeper.ZKUtil(111): regionserver:41571-0x1018fde83f30003, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2d55a1c3a545,41571,1731203373664 2024-11-10T01:49:33,712 WARN [RS:2;2d55a1c3a545:41571 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T01:49:33,712 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2d55a1c3a545,41571,1731203373664] 2024-11-10T01:49:33,712 INFO [RS:2;2d55a1c3a545:41571 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T01:49:33,712 DEBUG [RS:2;2d55a1c3a545:41571 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,41571,1731203373664 2024-11-10T01:49:33,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 207d1cdcf3d4218a6da14f099b34a4b8, server=2d55a1c3a545,34867,1731203370723 because future has completed 2024-11-10T01:49:33,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-10T01:49:33,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 207d1cdcf3d4218a6da14f099b34a4b8, server=2d55a1c3a545,34867,1731203370723 in 171 msec 2024-11-10T01:49:33,718 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T01:49:33,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T01:49:33,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731203371459, region=207d1cdcf3d4218a6da14f099b34a4b8, ASSIGN in 329 msec 2024-11-10T01:49:33,720 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 2d55a1c3a545,36983,1731203370694 after splitting done 2024-11-10T01:49:33,720 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T01:49:33,721 INFO [RS:2;2d55a1c3a545:41571 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T01:49:33,721 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:33,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 2d55a1c3a545,36983,1731203370694, splitWal=true, meta=false in 344 msec 2024-11-10T01:49:33,724 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T01:49:33,724 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T01:49:33,724 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2d55a1c3a545:0, corePoolSize=2, maxPoolSize=2 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2d55a1c3a545:0, corePoolSize=1, maxPoolSize=1 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2d55a1c3a545:0, corePoolSize=3, maxPoolSize=3 2024-11-10T01:49:33,725 DEBUG [RS:2;2d55a1c3a545:41571 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2d55a1c3a545:0, corePoolSize=3, maxPoolSize=3 2024-11-10T01:49:33,726 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:33,726 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:33,726 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:33,726 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:33,726 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:33,726 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,41571,1731203373664-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T01:49:33,742 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T01:49:33,742 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ChoreService(168): Chore ScheduledChore name=2d55a1c3a545,41571,1731203373664-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:33,743 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:33,743 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.Replication(171): 2d55a1c3a545,41571,1731203373664 started 2024-11-10T01:49:33,758 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T01:49:33,758 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.HRegionServer(1482): Serving as 2d55a1c3a545,41571,1731203373664, RpcServer on 2d55a1c3a545/172.17.0.2:41571, sessionid=0x1018fde83f30003 2024-11-10T01:49:33,758 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T01:49:33,758 DEBUG [RS:2;2d55a1c3a545:41571 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2d55a1c3a545,41571,1731203373664 2024-11-10T01:49:33,758 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2d55a1c3a545,41571,1731203373664' 2024-11-10T01:49:33,758 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T01:49:33,759 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T01:49:33,760 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T01:49:33,760 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T01:49:33,760 DEBUG [RS:2;2d55a1c3a545:41571 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2d55a1c3a545,41571,1731203373664 2024-11-10T01:49:33,760 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2d55a1c3a545,41571,1731203373664' 2024-11-10T01:49:33,760 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T01:49:33,760 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T01:49:33,761 DEBUG [RS:2;2d55a1c3a545:41571 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T01:49:33,761 INFO [RS:2;2d55a1c3a545:41571 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T01:49:33,761 INFO [RS:2;2d55a1c3a545:41571 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T01:49:33,862 INFO [RS:2;2d55a1c3a545:41571 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T01:49:33,862 INFO [RS:2;2d55a1c3a545:41571 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2d55a1c3a545%2C41571%2C1731203373664, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,41571,1731203373664, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs, maxLogs=55 2024-11-10T01:49:33,883 INFO [RS:2;2d55a1c3a545:41571 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/WALs/2d55a1c3a545,41571,1731203373664/2d55a1c3a545%2C41571%2C1731203373664.1731203373863 2024-11-10T01:49:33,884 DEBUG [RS:2;2d55a1c3a545:41571 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T01:49:34,258 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T01:49:34,258 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T01:49:34,258 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T01:49:34,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:34,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:34,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T01:49:34,259 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T01:49:34,259 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=223154735, stopped=false 2024-11-10T01:49:34,259 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2d55a1c3a545,38523,1731203370650 2024-11-10T01:49:34,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T01:49:34,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:34,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T01:49:34,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41571-0x1018fde83f30003, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T01:49:34,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:34,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41571-0x1018fde83f30003, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:34,261 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T01:49:34,262 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T01:49:34,262 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T01:49:34,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:34,262 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2d55a1c3a545,34867,1731203370723' ***** 2024-11-10T01:49:34,262 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T01:49:34,262 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T01:49:34,262 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41571-0x1018fde83f30003, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T01:49:34,262 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2d55a1c3a545,41571,1731203373664' ***** 2024-11-10T01:49:34,262 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T01:49:34,262 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T01:49:34,263 INFO [RS:1;2d55a1c3a545:34867 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T01:49:34,263 INFO [RS:2;2d55a1c3a545:41571 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T01:49:34,263 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T01:49:34,263 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T01:49:34,263 INFO [RS:1;2d55a1c3a545:34867 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.persistence 2024-11-10T01:49:34,263 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T01:49:34,263 INFO [RS:2;2d55a1c3a545:41571 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/13f12625-7381-a6cf-6505-95bcb331db44/bucket.persistence 2024-11-10T01:49:34,263 INFO [RS:1;2d55a1c3a545:34867 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T01:49:34,264 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T01:49:34,264 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T01:49:34,264 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T01:49:34,264 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T01:49:34,264 INFO [RS:2;2d55a1c3a545:41571 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T01:49:34,264 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T01:49:34,274 DEBUG [RS:1;2d55a1c3a545:34867 {}] bucket.BucketCache(1488): Thread RS:1;2d55a1c3a545:34867 started persisting bucket cache to file 2024-11-10T01:49:34,274 DEBUG [RS:1;2d55a1c3a545:34867 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T01:49:34,274 DEBUG [RS:1;2d55a1c3a545:34867 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T01:49:34,274 DEBUG [RS:2;2d55a1c3a545:41571 {}] bucket.BucketCache(1488): Thread RS:2;2d55a1c3a545:41571 started persisting bucket cache to file 2024-11-10T01:49:34,274 DEBUG [RS:2;2d55a1c3a545:41571 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T01:49:34,274 DEBUG [RS:2;2d55a1c3a545:41571 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T01:49:34,278 DEBUG [RS:1;2d55a1c3a545:34867 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T01:49:34,278 DEBUG [RS:1;2d55a1c3a545:34867 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731203374274 2024-11-10T01:49:34,278 DEBUG [RS:2;2d55a1c3a545:41571 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-10T01:49:34,278 DEBUG [RS:2;2d55a1c3a545:41571 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731203374274 2024-11-10T01:49:34,278 DEBUG [RS:1;2d55a1c3a545:34867 {}] bucket.BucketCache(1510): Thread RS:1;2d55a1c3a545:34867 finished persisting bucket cache to file, renaming 2024-11-10T01:49:34,278 DEBUG [RS:2;2d55a1c3a545:41571 {}] bucket.BucketCache(1510): Thread RS:2;2d55a1c3a545:41571 finished persisting bucket cache to file, renaming 2024-11-10T01:49:34,278 WARN [RS:2;2d55a1c3a545:41571 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-10T01:49:34,278 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T01:49:34,278 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T01:49:34,278 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T01:49:34,278 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T01:49:34,278 INFO [RS:2;2d55a1c3a545:41571 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T01:49:34,278 INFO [RS:2;2d55a1c3a545:41571 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T01:49:34,278 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.HRegionServer(959): stopping server 2d55a1c3a545,41571,1731203373664 2024-11-10T01:49:34,278 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T01:49:34,278 INFO [RS:1;2d55a1c3a545:34867 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T01:49:34,278 INFO [RS:1;2d55a1c3a545:34867 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T01:49:34,278 INFO [RS:2;2d55a1c3a545:41571 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2d55a1c3a545:41571. 2024-11-10T01:49:34,279 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(3091): Received CLOSE for 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:34,279 DEBUG [RS:2;2d55a1c3a545:41571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T01:49:34,279 DEBUG [RS:2;2d55a1c3a545:41571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:34,279 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.HRegionServer(976): stopping server 2d55a1c3a545,41571,1731203373664; all regions closed. 2024-11-10T01:49:34,279 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(959): stopping server 2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:34,279 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T01:49:34,279 INFO [RS:1;2d55a1c3a545:34867 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2d55a1c3a545:34867. 2024-11-10T01:49:34,279 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T01:49:34,279 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 207d1cdcf3d4218a6da14f099b34a4b8, disabling compactions & flushes 2024-11-10T01:49:34,279 DEBUG [RS:1;2d55a1c3a545:34867 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T01:49:34,279 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:34,279 DEBUG [RS:1;2d55a1c3a545:34867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:34,279 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:34,279 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. after waiting 0 ms 2024-11-10T01:49:34,279 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:34,279 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T01:49:34,279 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T01:49:34,280 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T01:49:34,280 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T01:49:34,281 DEBUG [RS:2;2d55a1c3a545:41571 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs 2024-11-10T01:49:34,281 INFO [RS:2;2d55a1c3a545:41571 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2d55a1c3a545%2C41571%2C1731203373664:(num 1731203373863) 2024-11-10T01:49:34,281 DEBUG [RS:2;2d55a1c3a545:41571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:34,281 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T01:49:34,281 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T01:49:34,281 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.ChoreService(370): Chore service for: regionserver/2d55a1c3a545:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T01:49:34,281 DEBUG [StoreFileCloser-207d1cdcf3d4218a6da14f099b34a4b8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/family/36a75db902e64d70be28697ff520366d 2024-11-10T01:49:34,282 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T01:49:34,282 INFO [regionserver/2d55a1c3a545:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T01:49:34,282 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T01:49:34,282 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T01:49:34,282 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T01:49:34,282 INFO [RS:2;2d55a1c3a545:41571 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41571 2024-11-10T01:49:34,282 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T01:49:34,282 DEBUG [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 207d1cdcf3d4218a6da14f099b34a4b8=testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8.} 2024-11-10T01:49:34,282 DEBUG [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 207d1cdcf3d4218a6da14f099b34a4b8 2024-11-10T01:49:34,282 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T01:49:34,282 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T01:49:34,283 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T01:49:34,283 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T01:49:34,283 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T01:49:34,283 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T01:49:34,283 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-10T01:49:34,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41571-0x1018fde83f30003, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2d55a1c3a545,41571,1731203373664 2024-11-10T01:49:34,284 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T01:49:34,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T01:49:34,286 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2d55a1c3a545,41571,1731203373664] 2024-11-10T01:49:34,287 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/default/testBlockEvictionOnGracefulStop1731203371459/207d1cdcf3d4218a6da14f099b34a4b8/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-10T01:49:34,287 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2d55a1c3a545,41571,1731203373664 already deleted, retry=false 2024-11-10T01:49:34,287 INFO [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:34,288 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2d55a1c3a545,41571,1731203373664 expired; onlineServers=1 2024-11-10T01:49:34,288 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 207d1cdcf3d4218a6da14f099b34a4b8: Waiting for close lock at 1731203374279Running coprocessor pre-close hooks at 1731203374279Disabling compacts and flushes for region at 1731203374279Disabling writes for close at 1731203374279Writing region close event to WAL at 1731203374282 (+3 ms)Running coprocessor post-close hooks at 1731203374287 (+5 ms)Closed at 1731203374287 2024-11-10T01:49:34,288 DEBUG [RS_CLOSE_REGION-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8. 2024-11-10T01:49:34,294 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a00fe50a25394e218764aba566e71566 2024-11-10T01:49:34,308 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/info/a00fe50a25394e218764aba566e71566 is 209, key is testBlockEvictionOnGracefulStop1731203371459,,1731203371463.207d1cdcf3d4218a6da14f099b34a4b8./info:regioninfo/1731203373711/Put/seqid=0 2024-11-10T01:49:34,309 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/info/a00fe50a25394e218764aba566e71566 2024-11-10T01:49:34,317 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a473a26b366a476aba5d94d4c1830fc8 2024-11-10T01:49:34,322 INFO [regionserver/2d55a1c3a545:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T01:49:34,328 INFO [regionserver/2d55a1c3a545:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T01:49:34,332 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/ns/a473a26b366a476aba5d94d4c1830fc8 is 43, key is default/ns:d/1731203371419/Put/seqid=0 2024-11-10T01:49:34,333 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/ns/a473a26b366a476aba5d94d4c1830fc8 2024-11-10T01:49:34,342 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b92f235f648b4fcdb8c40284674fe88f 2024-11-10T01:49:34,355 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/table/b92f235f648b4fcdb8c40284674fe88f is 80, key is testBlockEvictionOnGracefulStop1731203371459/table:state/1731203371842/Put/seqid=0 2024-11-10T01:49:34,356 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/table/b92f235f648b4fcdb8c40284674fe88f 2024-11-10T01:49:34,358 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/info/a00fe50a25394e218764aba566e71566 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/info/a00fe50a25394e218764aba566e71566 2024-11-10T01:49:34,360 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/info/a00fe50a25394e218764aba566e71566, delay=900 ms 2024-11-10T01:49:34,361 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/info/a00fe50a25394e218764aba566e71566, entries=14, sequenceid=13, filesize=7.5 K 2024-11-10T01:49:34,361 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/ns/a473a26b366a476aba5d94d4c1830fc8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/ns/a473a26b366a476aba5d94d4c1830fc8 2024-11-10T01:49:34,362 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/ns/a473a26b366a476aba5d94d4c1830fc8, delay=974 ms 2024-11-10T01:49:34,364 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/ns/a473a26b366a476aba5d94d4c1830fc8, entries=2, sequenceid=13, filesize=5.0 K 2024-11-10T01:49:34,364 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/.tmp/table/b92f235f648b4fcdb8c40284674fe88f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/table/b92f235f648b4fcdb8c40284674fe88f 2024-11-10T01:49:34,365 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/table/b92f235f648b4fcdb8c40284674fe88f, delay=985 ms 2024-11-10T01:49:34,367 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/table/b92f235f648b4fcdb8c40284674fe88f, entries=2, sequenceid=13, filesize=5.3 K 2024-11-10T01:49:34,367 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 84ms, sequenceid=13, compaction requested=false 2024-11-10T01:49:34,368 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/info/a00fe50a25394e218764aba566e71566 2024-11-10T01:49:34,369 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/ns/a473a26b366a476aba5d94d4c1830fc8 2024-11-10T01:49:34,369 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/table/b92f235f648b4fcdb8c40284674fe88f 2024-11-10T01:49:34,374 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-10T01:49:34,374 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T01:49:34,375 INFO [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T01:49:34,375 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731203374282Running coprocessor pre-close hooks at 1731203374282Disabling compacts and flushes for region at 1731203374282Disabling writes for close at 1731203374283 (+1 ms)Obtaining lock to block concurrent updates at 1731203374283Preparing flush snapshotting stores in 1588230740 at 1731203374283Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731203374283Flushing stores of hbase:meta,,1.1588230740 at 1731203374284 (+1 ms)Flushing 1588230740/info: creating writer at 1731203374284Flushing 1588230740/info: appending metadata at 1731203374307 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731203374307Flushing 1588230740/ns: creating writer at 1731203374311 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731203374332 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731203374332Flushing 1588230740/table: creating writer at 1731203374336 (+4 ms)Flushing 1588230740/table: appending metadata at 1731203374354 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731203374354Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32c591b7: reopening flushed file at 1731203374358 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f50ccc1: reopening flushed file at 1731203374361 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@744c9937: reopening flushed file at 1731203374364 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 84ms, sequenceid=13, compaction requested=false at 1731203374368 (+4 ms)Writing region close event to WAL at 1731203374369 (+1 ms)Running coprocessor post-close hooks at 1731203374374 (+5 ms)Closed at 1731203374375 (+1 ms) 2024-11-10T01:49:34,375 DEBUG [RS_CLOSE_META-regionserver/2d55a1c3a545:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T01:49:34,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41571-0x1018fde83f30003, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:34,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41571-0x1018fde83f30003, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:34,386 INFO [RS:2;2d55a1c3a545:41571 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T01:49:34,386 INFO [RS:2;2d55a1c3a545:41571 {}] regionserver.HRegionServer(1031): Exiting; stopping=2d55a1c3a545,41571,1731203373664; zookeeper connection closed. 2024-11-10T01:49:34,386 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@66cd07ac {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@66cd07ac 2024-11-10T01:49:34,483 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(976): stopping server 2d55a1c3a545,34867,1731203370723; all regions closed. 2024-11-10T01:49:34,485 DEBUG [RS:1;2d55a1c3a545:34867 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs 2024-11-10T01:49:34,485 INFO [RS:1;2d55a1c3a545:34867 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2d55a1c3a545%2C34867%2C1731203370723.meta:.meta(num 1731203371363) 2024-11-10T01:49:34,486 DEBUG [RS:1;2d55a1c3a545:34867 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/oldWALs 2024-11-10T01:49:34,486 INFO [RS:1;2d55a1c3a545:34867 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2d55a1c3a545%2C34867%2C1731203370723:(num 1731203371160) 2024-11-10T01:49:34,486 DEBUG [RS:1;2d55a1c3a545:34867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T01:49:34,486 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T01:49:34,486 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T01:49:34,487 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.ChoreService(370): Chore service for: regionserver/2d55a1c3a545:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T01:49:34,487 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T01:49:34,487 INFO [regionserver/2d55a1c3a545:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T01:49:34,487 INFO [RS:1;2d55a1c3a545:34867 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34867 2024-11-10T01:49:34,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2d55a1c3a545,34867,1731203370723 2024-11-10T01:49:34,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T01:49:34,489 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T01:49:34,491 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2d55a1c3a545,34867,1731203370723] 2024-11-10T01:49:34,493 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2d55a1c3a545,34867,1731203370723 already deleted, retry=false 2024-11-10T01:49:34,493 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2d55a1c3a545,34867,1731203370723 expired; onlineServers=0 2024-11-10T01:49:34,493 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2d55a1c3a545,38523,1731203370650' ***** 2024-11-10T01:49:34,493 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T01:49:34,493 INFO [M:0;2d55a1c3a545:38523 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T01:49:34,493 INFO [M:0;2d55a1c3a545:38523 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T01:49:34,493 DEBUG [M:0;2d55a1c3a545:38523 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T01:49:34,493 DEBUG [M:0;2d55a1c3a545:38523 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T01:49:34,493 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster-HFileCleaner.large.0-1731203370970 {}] cleaner.HFileCleaner(306): Exit Thread[master/2d55a1c3a545:0:becomeActiveMaster-HFileCleaner.large.0-1731203370970,5,FailOnTimeoutGroup] 2024-11-10T01:49:34,493 DEBUG [master/2d55a1c3a545:0:becomeActiveMaster-HFileCleaner.small.0-1731203370971 {}] cleaner.HFileCleaner(306): Exit Thread[master/2d55a1c3a545:0:becomeActiveMaster-HFileCleaner.small.0-1731203370971,5,FailOnTimeoutGroup] 2024-11-10T01:49:34,493 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T01:49:34,494 INFO [M:0;2d55a1c3a545:38523 {}] hbase.ChoreService(370): Chore service for: master/2d55a1c3a545:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T01:49:34,494 INFO [M:0;2d55a1c3a545:38523 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T01:49:34,494 DEBUG [M:0;2d55a1c3a545:38523 {}] master.HMaster(1795): Stopping service threads 2024-11-10T01:49:34,494 INFO [M:0;2d55a1c3a545:38523 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T01:49:34,494 INFO [M:0;2d55a1c3a545:38523 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T01:49:34,494 INFO [M:0;2d55a1c3a545:38523 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T01:49:34,494 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T01:49:34,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T01:49:34,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T01:49:34,495 DEBUG [M:0;2d55a1c3a545:38523 {}] zookeeper.ZKUtil(347): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T01:49:34,496 WARN [M:0;2d55a1c3a545:38523 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T01:49:34,496 INFO [M:0;2d55a1c3a545:38523 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/.lastflushedseqids 2024-11-10T01:49:34,502 INFO [M:0;2d55a1c3a545:38523 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T01:49:34,503 INFO [M:0;2d55a1c3a545:38523 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T01:49:34,503 DEBUG [M:0;2d55a1c3a545:38523 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T01:49:34,503 INFO [M:0;2d55a1c3a545:38523 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:34,503 DEBUG [M:0;2d55a1c3a545:38523 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:34,503 DEBUG [M:0;2d55a1c3a545:38523 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T01:49:34,503 DEBUG [M:0;2d55a1c3a545:38523 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:34,503 INFO [M:0;2d55a1c3a545:38523 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-11-10T01:49:34,513 DEBUG [M:0;2d55a1c3a545:38523 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2e61ca65535e4d988d2eb6d2dbf26257 2024-11-10T01:49:34,526 DEBUG [M:0;2d55a1c3a545:38523 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2e61ca65535e4d988d2eb6d2dbf26257 is 82, key is hbase:meta,,1/info:regioninfo/1731203371403/Put/seqid=0 2024-11-10T01:49:34,526 INFO [M:0;2d55a1c3a545:38523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2e61ca65535e4d988d2eb6d2dbf26257 2024-11-10T01:49:34,534 DEBUG [M:0;2d55a1c3a545:38523 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e5625dcf78ed4e00a5080ed421792043 2024-11-10T01:49:34,547 DEBUG [M:0;2d55a1c3a545:38523 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e5625dcf78ed4e00a5080ed421792043 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731203371847/Put/seqid=0 2024-11-10T01:49:34,548 INFO [M:0;2d55a1c3a545:38523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e5625dcf78ed4e00a5080ed421792043 2024-11-10T01:49:34,555 DEBUG [M:0;2d55a1c3a545:38523 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fa8db9e542f143c2af0b0d903ca8aa8e 2024-11-10T01:49:34,569 DEBUG [M:0;2d55a1c3a545:38523 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fa8db9e542f143c2af0b0d903ca8aa8e is 69, key is 2d55a1c3a545,34867,1731203370723/rs:state/1731203370978/Put/seqid=0 2024-11-10T01:49:34,569 INFO [M:0;2d55a1c3a545:38523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fa8db9e542f143c2af0b0d903ca8aa8e 2024-11-10T01:49:34,571 INFO [M:0;2d55a1c3a545:38523 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa8db9e542f143c2af0b0d903ca8aa8e 2024-11-10T01:49:34,571 DEBUG [M:0;2d55a1c3a545:38523 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2e61ca65535e4d988d2eb6d2dbf26257 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2e61ca65535e4d988d2eb6d2dbf26257 2024-11-10T01:49:34,573 INFO [M:0;2d55a1c3a545:38523 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2e61ca65535e4d988d2eb6d2dbf26257, entries=8, sequenceid=83, filesize=5.5 K 2024-11-10T01:49:34,573 DEBUG [M:0;2d55a1c3a545:38523 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e5625dcf78ed4e00a5080ed421792043 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e5625dcf78ed4e00a5080ed421792043 2024-11-10T01:49:34,575 INFO [M:0;2d55a1c3a545:38523 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e5625dcf78ed4e00a5080ed421792043, entries=9, sequenceid=83, filesize=6.4 K 2024-11-10T01:49:34,575 DEBUG [M:0;2d55a1c3a545:38523 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fa8db9e542f143c2af0b0d903ca8aa8e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fa8db9e542f143c2af0b0d903ca8aa8e 2024-11-10T01:49:34,577 INFO [M:0;2d55a1c3a545:38523 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa8db9e542f143c2af0b0d903ca8aa8e 2024-11-10T01:49:34,577 INFO [M:0;2d55a1c3a545:38523 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0195f455-f095-071d-6629-45636d58fa6f/80db9358-9790-6b47-481a-21991dae5f4b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fa8db9e542f143c2af0b0d903ca8aa8e, entries=3, sequenceid=83, filesize=5.3 K 2024-11-10T01:49:34,578 INFO [M:0;2d55a1c3a545:38523 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 74ms, sequenceid=83, compaction requested=false 2024-11-10T01:49:34,579 INFO [M:0;2d55a1c3a545:38523 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T01:49:34,579 DEBUG [M:0;2d55a1c3a545:38523 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731203374503Disabling compacts and flushes for region at 1731203374503Disabling writes for close at 1731203374503Obtaining lock to block concurrent updates at 1731203374503Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731203374503Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1731203374504 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731203374504Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731203374504Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731203374525 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731203374525Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731203374528 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731203374547 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731203374547Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731203374549 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731203374568 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731203374568Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e3295f6: reopening flushed file at 1731203374571 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e9ec20e: reopening flushed file at 1731203374573 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fbd6fc5: reopening flushed file at 1731203374575 (+2 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 74ms, sequenceid=83, compaction requested=false at 1731203374578 (+3 ms)Writing region close event to WAL at 1731203374579 (+1 ms)Closed at 1731203374579 2024-11-10T01:49:34,580 INFO [M:0;2d55a1c3a545:38523 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T01:49:34,580 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T01:49:34,580 INFO [M:0;2d55a1c3a545:38523 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38523 2024-11-10T01:49:34,581 INFO [M:0;2d55a1c3a545:38523 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T01:49:34,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:34,591 INFO [RS:1;2d55a1c3a545:34867 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T01:49:34,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1018fde83f30002, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:34,591 INFO [RS:1;2d55a1c3a545:34867 {}] regionserver.HRegionServer(1031): Exiting; stopping=2d55a1c3a545,34867,1731203370723; zookeeper connection closed. 2024-11-10T01:49:34,592 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@76a5c59 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@76a5c59 2024-11-10T01:49:34,592 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-10T01:49:34,683 INFO [M:0;2d55a1c3a545:38523 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T01:49:34,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:34,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1018fde83f30000, quorum=127.0.0.1:53132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T01:49:34,686 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T01:49:34,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T01:49:34,687 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:49:34,694 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=139 (was 151), ProcessCount=11 (was 11), AvailableMemoryMB=8134 (was 8204)