2024-12-04 22:44:36,778 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-04 22:44:36,795 main DEBUG Took 0.014956 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 22:44:36,795 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 22:44:36,796 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 22:44:36,796 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 22:44:36,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,803 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 22:44:36,814 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,816 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,816 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,817 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,817 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,817 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,818 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,818 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,819 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,819 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,820 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,820 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,820 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,821 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,822 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,822 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,822 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,823 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,824 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,824 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 22:44:36,824 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,824 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 22:44:36,826 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 22:44:36,827 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 22:44:36,829 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 22:44:36,830 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 22:44:36,831 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 22:44:36,831 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 22:44:36,839 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 22:44:36,842 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 22:44:36,843 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 22:44:36,843 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 22:44:36,844 main DEBUG createAppenders(={Console}) 2024-12-04 22:44:36,845 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-04 22:44:36,845 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-04 22:44:36,845 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-04 22:44:36,846 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 22:44:36,846 main DEBUG OutputStream closed 2024-12-04 22:44:36,847 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 22:44:36,847 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 22:44:36,847 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-04 22:44:36,921 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 22:44:36,924 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 22:44:36,925 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 22:44:36,926 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 22:44:36,927 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 22:44:36,927 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 22:44:36,928 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 22:44:36,928 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 22:44:36,928 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 22:44:36,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 22:44:36,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 22:44:36,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 22:44:36,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 22:44:36,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 22:44:36,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 22:44:36,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 22:44:36,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 22:44:36,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 22:44:36,934 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 22:44:36,934 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-04 22:44:36,935 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 22:44:36,936 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-04T22:44:36,951 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-04 22:44:36,954 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 22:44:36,955 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T22:44:37,220 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2 2024-12-04T22:44:37,247 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=11611 2024-12-04T22:44:37,343 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T22:44:37,434 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:37,449 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/cluster_53577ec1-7690-d9bf-2b64-184fb14e9215, deleteOnExit=true 2024-12-04T22:44:37,537 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/cluster_53577ec1-7690-d9bf-2b64-184fb14e9215/zookeeper_0, clientPort=52727, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/cluster_53577ec1-7690-d9bf-2b64-184fb14e9215/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/cluster_53577ec1-7690-d9bf-2b64-184fb14e9215/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T22:44:37,548 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52727 2024-12-04T22:44:37,549 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:37,625 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b with version=8 2024-12-04T22:44:37,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/hbase-staging 2024-12-04T22:44:37,711 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T22:44:37,967 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9840c0493aab:0 server-side Connection retries=45 2024-12-04T22:44:37,975 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:37,976 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:37,983 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T22:44:37,983 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:37,983 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T22:44:38,099 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T22:44:38,183 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T22:44:38,224 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T22:44:38,234 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T22:44:38,238 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T22:44:38,272 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12040 (auto-detected) 2024-12-04T22:44:38,274 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T22:44:38,295 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41337 2024-12-04T22:44:38,316 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41337 connecting to ZooKeeper ensemble=127.0.0.1:52727 2024-12-04T22:44:38,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:413370x0, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T22:44:38,453 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41337-0x101a284c1bc0000 connected 2024-12-04T22:44:38,988 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:38,989 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:38,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T22:44:39,005 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b, hbase.cluster.distributed=false 2024-12-04T22:44:39,030 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T22:44:39,035 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41337 2024-12-04T22:44:39,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41337 2024-12-04T22:44:39,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41337 2024-12-04T22:44:39,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41337 2024-12-04T22:44:39,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41337 2024-12-04T22:44:39,106 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9840c0493aab:0 server-side Connection retries=45 2024-12-04T22:44:39,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:39,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:39,109 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T22:44:39,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:39,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T22:44:39,113 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T22:44:39,116 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T22:44:39,118 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40395 2024-12-04T22:44:39,120 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40395 connecting to ZooKeeper ensemble=127.0.0.1:52727 2024-12-04T22:44:39,121 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:39,122 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:39,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403950x0, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T22:44:39,145 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40395-0x101a284c1bc0001 connected 2024-12-04T22:44:39,145 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T22:44:39,153 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T22:44:39,168 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/bucket.cache 2024-12-04T22:44:39,174 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-04T22:44:39,176 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-04T22:44:39,178 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-04T22:44:39,180 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-04T22:44:39,181 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-04T22:44:39,188 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-04T22:44:39,191 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-04T22:44:39,191 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/bucket.persistence is complete. 2024-12-04T22:44:39,193 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T22:44:39,196 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T22:44:39,201 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T22:44:39,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40395 2024-12-04T22:44:39,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40395 2024-12-04T22:44:39,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40395 2024-12-04T22:44:39,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40395 2024-12-04T22:44:39,209 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40395 2024-12-04T22:44:39,235 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9840c0493aab:0 server-side Connection retries=45 2024-12-04T22:44:39,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:39,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:39,237 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T22:44:39,237 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:39,237 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T22:44:39,238 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T22:44:39,238 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T22:44:39,239 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43247 2024-12-04T22:44:39,242 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43247 connecting to ZooKeeper ensemble=127.0.0.1:52727 2024-12-04T22:44:39,243 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:39,244 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:39,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432470x0, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T22:44:39,271 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:432470x0, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T22:44:39,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43247-0x101a284c1bc0002 connected 2024-12-04T22:44:39,272 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T22:44:39,273 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/bucket.cache 2024-12-04T22:44:39,273 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-04T22:44:39,274 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-04T22:44:39,274 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-04T22:44:39,275 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-04T22:44:39,275 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-04T22:44:39,275 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-04T22:44:39,276 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-04T22:44:39,276 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T22:44:39,276 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/bucket.persistence is complete. 2024-12-04T22:44:39,278 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T22:44:39,281 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T22:44:39,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43247 2024-12-04T22:44:39,287 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43247 2024-12-04T22:44:39,288 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43247 2024-12-04T22:44:39,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43247 2024-12-04T22:44:39,293 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43247 2024-12-04T22:44:39,316 DEBUG [M:0;9840c0493aab:41337 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9840c0493aab:41337 2024-12-04T22:44:39,317 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9840c0493aab,41337,1733352277793 2024-12-04T22:44:39,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T22:44:39,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T22:44:39,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T22:44:39,341 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9840c0493aab,41337,1733352277793 2024-12-04T22:44:39,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:39,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T22:44:39,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T22:44:39,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:39,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:39,377 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T22:44:39,378 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9840c0493aab,41337,1733352277793 from backup master directory 2024-12-04T22:44:39,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T22:44:39,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T22:44:39,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9840c0493aab,41337,1733352277793 2024-12-04T22:44:39,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T22:44:39,392 WARN [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T22:44:39,392 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9840c0493aab,41337,1733352277793 2024-12-04T22:44:39,394 INFO [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T22:44:39,396 INFO [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T22:44:39,404 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/hbase.id] with ID: f9f1c9b0-bd09-46c1-9c4f-de946f32d48d 2024-12-04T22:44:39,404 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/.tmp/hbase.id 2024-12-04T22:44:39,410 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/hbase.id] 2024-12-04T22:44:39,456 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:39,462 INFO [master/9840c0493aab:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T22:44:39,488 INFO [master/9840c0493aab:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-12-04T22:44:39,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:39,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:39,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:39,533 INFO [master/9840c0493aab:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T22:44:39,536 INFO [master/9840c0493aab:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T22:44:39,579 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T22:44:39,587 INFO [master/9840c0493aab:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T22:44:39,680 INFO [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store 2024-12-04T22:44:39,704 INFO [master/9840c0493aab:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T22:44:39,707 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:39,710 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T22:44:39,710 INFO [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:39,710 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:39,713 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T22:44:39,713 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:39,713 INFO [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:39,715 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733352279709Disabling compacts and flushes for region at 1733352279709Disabling writes for close at 1733352279713 (+4 ms)Writing region close event to WAL at 1733352279713Closed at 1733352279713 2024-12-04T22:44:39,716 WARN [master/9840c0493aab:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/.initializing 2024-12-04T22:44:39,716 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/WALs/9840c0493aab,41337,1733352277793 2024-12-04T22:44:39,730 INFO [master/9840c0493aab:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T22:44:39,744 INFO [master/9840c0493aab:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9840c0493aab%2C41337%2C1733352277793, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/WALs/9840c0493aab,41337,1733352277793, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/oldWALs, maxLogs=10 2024-12-04T22:44:39,779 INFO [master/9840c0493aab:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/WALs/9840c0493aab,41337,1733352277793/9840c0493aab%2C41337%2C1733352277793.1733352279749 2024-12-04T22:44:39,780 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T22:44:39,781 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T22:44:39,781 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:39,785 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:39,786 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:39,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:39,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T22:44:39,862 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:39,864 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:39,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:39,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T22:44:39,870 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:39,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T22:44:39,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:39,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T22:44:39,877 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:39,877 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T22:44:39,878 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:39,882 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T22:44:39,883 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:39,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T22:44:39,884 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:39,887 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:39,888 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:39,890 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:39,891 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:39,893 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T22:44:39,895 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:39,901 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T22:44:39,902 INFO [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59558575, jitterRate=-0.11250807344913483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T22:44:39,909 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733352279800Initializing all the Stores at 1733352279801 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352279801Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352279802 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352279803 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352279803Cleaning up temporary data from old regions at 1733352279891 (+88 ms)Region opened successfully at 1733352279908 (+17 ms) 2024-12-04T22:44:39,911 INFO [master/9840c0493aab:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T22:44:39,947 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d7128c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9840c0493aab/172.17.0.2:0 2024-12-04T22:44:39,988 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T22:44:40,005 INFO [master/9840c0493aab:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T22:44:40,005 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T22:44:40,010 INFO [master/9840c0493aab:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T22:44:40,012 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T22:44:40,021 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-12-04T22:44:40,021 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T22:44:40,067 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T22:44:40,083 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T22:44:40,106 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T22:44:40,109 INFO [master/9840c0493aab:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T22:44:40,111 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T22:44:40,127 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T22:44:40,130 INFO [master/9840c0493aab:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T22:44:40,136 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T22:44:40,149 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T22:44:40,154 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T22:44:40,165 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T22:44:40,193 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T22:44:40,206 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T22:44:40,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T22:44:40,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T22:44:40,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T22:44:40,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:40,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:40,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:40,231 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9840c0493aab,41337,1733352277793, sessionid=0x101a284c1bc0000, setting cluster-up flag (Was=false) 2024-12-04T22:44:40,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:40,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:40,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:40,296 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T22:44:40,303 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9840c0493aab,41337,1733352277793 2024-12-04T22:44:40,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:40,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:40,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:40,359 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T22:44:40,362 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9840c0493aab,41337,1733352277793 2024-12-04T22:44:40,371 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T22:44:40,397 INFO [RS:0;9840c0493aab:40395 {}] regionserver.HRegionServer(746): ClusterId : f9f1c9b0-bd09-46c1-9c4f-de946f32d48d 2024-12-04T22:44:40,397 INFO [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(746): ClusterId : f9f1c9b0-bd09-46c1-9c4f-de946f32d48d 2024-12-04T22:44:40,399 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T22:44:40,399 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T22:44:40,413 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T22:44:40,413 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T22:44:40,414 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T22:44:40,414 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T22:44:40,423 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T22:44:40,423 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T22:44:40,424 DEBUG [RS:1;9840c0493aab:43247 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@357be385, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9840c0493aab/172.17.0.2:0 2024-12-04T22:44:40,424 DEBUG [RS:0;9840c0493aab:40395 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f1e080, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9840c0493aab/172.17.0.2:0 2024-12-04T22:44:40,439 DEBUG [RS:0;9840c0493aab:40395 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9840c0493aab:40395 2024-12-04T22:44:40,440 DEBUG [RS:1;9840c0493aab:43247 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9840c0493aab:43247 2024-12-04T22:44:40,442 INFO [RS:0;9840c0493aab:40395 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T22:44:40,442 INFO [RS:1;9840c0493aab:43247 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T22:44:40,442 INFO [RS:0;9840c0493aab:40395 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T22:44:40,442 INFO [RS:1;9840c0493aab:43247 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T22:44:40,442 DEBUG [RS:0;9840c0493aab:40395 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T22:44:40,442 DEBUG [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T22:44:40,445 INFO [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(2659): reportForDuty to master=9840c0493aab,41337,1733352277793 with port=43247, startcode=1733352279235 2024-12-04T22:44:40,445 INFO [RS:0;9840c0493aab:40395 {}] regionserver.HRegionServer(2659): reportForDuty to master=9840c0493aab,41337,1733352277793 with port=40395, startcode=1733352279076 2024-12-04T22:44:40,450 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T22:44:40,458 DEBUG [RS:0;9840c0493aab:40395 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T22:44:40,458 DEBUG [RS:1;9840c0493aab:43247 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T22:44:40,461 INFO [master/9840c0493aab:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T22:44:40,469 INFO [master/9840c0493aab:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T22:44:40,476 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9840c0493aab,41337,1733352277793 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T22:44:40,483 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9840c0493aab:0, corePoolSize=5, maxPoolSize=5 2024-12-04T22:44:40,484 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9840c0493aab:0, corePoolSize=5, maxPoolSize=5 2024-12-04T22:44:40,484 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9840c0493aab:0, corePoolSize=5, maxPoolSize=5 2024-12-04T22:44:40,484 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9840c0493aab:0, corePoolSize=5, maxPoolSize=5 2024-12-04T22:44:40,484 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9840c0493aab:0, corePoolSize=10, maxPoolSize=10 2024-12-04T22:44:40,485 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,488 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9840c0493aab:0, corePoolSize=2, maxPoolSize=2 2024-12-04T22:44:40,488 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,496 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733352310496 2024-12-04T22:44:40,498 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T22:44:40,499 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T22:44:40,499 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T22:44:40,499 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T22:44:40,505 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T22:44:40,507 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:40,507 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T22:44:40,507 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T22:44:40,508 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T22:44:40,507 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T22:44:40,510 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,513 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T22:44:40,515 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T22:44:40,515 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T22:44:40,519 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T22:44:40,519 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T22:44:40,521 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9840c0493aab:0:becomeActiveMaster-HFileCleaner.large.0-1733352280520,5,FailOnTimeoutGroup] 2024-12-04T22:44:40,522 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9840c0493aab:0:becomeActiveMaster-HFileCleaner.small.0-1733352280521,5,FailOnTimeoutGroup] 2024-12-04T22:44:40,522 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,522 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T22:44:40,523 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,523 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,528 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T22:44:40,529 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b 2024-12-04T22:44:40,531 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57437, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T22:44:40,531 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53589, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T22:44:40,538 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:40,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-04T22:44:40,539 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41337 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9840c0493aab,43247,1733352279235 2024-12-04T22:44:40,543 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41337 {}] master.ServerManager(517): Registering regionserver=9840c0493aab,43247,1733352279235 2024-12-04T22:44:40,548 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T22:44:40,548 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:40,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:40,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-04T22:44:40,553 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T22:44:40,553 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:40,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:40,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-04T22:44:40,557 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41337 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9840c0493aab,40395,1733352279076 2024-12-04T22:44:40,557 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41337 {}] master.ServerManager(517): Registering regionserver=9840c0493aab,40395,1733352279076 2024-12-04T22:44:40,560 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T22:44:40,560 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:40,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:40,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-04T22:44:40,561 DEBUG [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b 2024-12-04T22:44:40,561 DEBUG [RS:0;9840c0493aab:40395 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b 2024-12-04T22:44:40,562 DEBUG [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-04T22:44:40,562 DEBUG [RS:0;9840c0493aab:40395 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-04T22:44:40,562 DEBUG [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T22:44:40,562 DEBUG [RS:0;9840c0493aab:40395 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T22:44:40,569 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T22:44:40,569 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:40,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:40,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T22:44:40,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740 2024-12-04T22:44:40,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740 2024-12-04T22:44:40,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T22:44:40,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T22:44:40,572 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T22:44:40,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T22:44:40,583 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T22:44:40,584 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69767970, jitterRate=0.039623767137527466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T22:44:40,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733352280538Initializing all the Stores at 1733352280538Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352280538Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352280538Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352280538Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352280538Cleaning up temporary data from old regions at 1733352280571 (+33 ms)Region opened successfully at 1733352280586 (+15 ms) 2024-12-04T22:44:40,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T22:44:40,586 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T22:44:40,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T22:44:40,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T22:44:40,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T22:44:40,588 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T22:44:40,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733352280586Disabling compacts and flushes for region at 1733352280586Disabling writes for close at 1733352280587 (+1 ms)Writing region close event to WAL at 1733352280588 (+1 ms)Closed at 1733352280588 2024-12-04T22:44:40,590 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T22:44:40,590 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T22:44:40,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T22:44:40,603 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T22:44:40,605 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T22:44:40,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T22:44:40,608 DEBUG [RS:0;9840c0493aab:40395 {}] zookeeper.ZKUtil(111): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9840c0493aab,40395,1733352279076 2024-12-04T22:44:40,608 DEBUG [RS:1;9840c0493aab:43247 {}] zookeeper.ZKUtil(111): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9840c0493aab,43247,1733352279235 2024-12-04T22:44:40,608 WARN [RS:0;9840c0493aab:40395 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T22:44:40,608 WARN [RS:1;9840c0493aab:43247 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T22:44:40,608 INFO [RS:0;9840c0493aab:40395 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T22:44:40,608 INFO [RS:1;9840c0493aab:43247 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T22:44:40,608 DEBUG [RS:0;9840c0493aab:40395 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,40395,1733352279076 2024-12-04T22:44:40,608 DEBUG [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,43247,1733352279235 2024-12-04T22:44:40,611 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9840c0493aab,43247,1733352279235] 2024-12-04T22:44:40,611 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9840c0493aab,40395,1733352279076] 2024-12-04T22:44:40,633 INFO [RS:1;9840c0493aab:43247 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T22:44:40,633 INFO [RS:0;9840c0493aab:40395 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T22:44:40,652 INFO [RS:0;9840c0493aab:40395 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T22:44:40,653 INFO [RS:1;9840c0493aab:43247 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T22:44:40,660 INFO [RS:0;9840c0493aab:40395 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T22:44:40,660 INFO [RS:1;9840c0493aab:43247 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T22:44:40,661 INFO [RS:0;9840c0493aab:40395 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,661 INFO [RS:1;9840c0493aab:43247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,662 INFO [RS:0;9840c0493aab:40395 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T22:44:40,664 INFO [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T22:44:40,668 INFO [RS:0;9840c0493aab:40395 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T22:44:40,668 INFO [RS:1;9840c0493aab:43247 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T22:44:40,670 INFO [RS:0;9840c0493aab:40395 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,670 INFO [RS:1;9840c0493aab:43247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,670 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,670 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,670 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,670 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,671 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,671 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,671 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,671 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,671 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,671 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,671 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9840c0493aab:0, corePoolSize=2, maxPoolSize=2 2024-12-04T22:44:40,671 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9840c0493aab:0, corePoolSize=2, maxPoolSize=2 2024-12-04T22:44:40,671 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,671 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,671 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,672 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,672 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,672 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,672 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,672 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,672 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,672 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,672 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,672 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:40,672 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9840c0493aab:0, corePoolSize=3, maxPoolSize=3 2024-12-04T22:44:40,672 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9840c0493aab:0, corePoolSize=3, maxPoolSize=3 2024-12-04T22:44:40,672 DEBUG [RS:0;9840c0493aab:40395 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9840c0493aab:0, corePoolSize=3, maxPoolSize=3 2024-12-04T22:44:40,673 DEBUG [RS:1;9840c0493aab:43247 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9840c0493aab:0, corePoolSize=3, maxPoolSize=3 2024-12-04T22:44:40,685 INFO [RS:0;9840c0493aab:40395 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,685 INFO [RS:1;9840c0493aab:43247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,685 INFO [RS:0;9840c0493aab:40395 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,685 INFO [RS:1;9840c0493aab:43247 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,686 INFO [RS:0;9840c0493aab:40395 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,686 INFO [RS:0;9840c0493aab:40395 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,686 INFO [RS:0;9840c0493aab:40395 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,686 INFO [RS:0;9840c0493aab:40395 {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,40395,1733352279076-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T22:44:40,686 INFO [RS:1;9840c0493aab:43247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,686 INFO [RS:1;9840c0493aab:43247 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,687 INFO [RS:1;9840c0493aab:43247 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,687 INFO [RS:1;9840c0493aab:43247 {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,43247,1733352279235-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T22:44:40,716 INFO [RS:0;9840c0493aab:40395 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T22:44:40,716 INFO [RS:1;9840c0493aab:43247 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T22:44:40,719 INFO [RS:1;9840c0493aab:43247 {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,43247,1733352279235-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,719 INFO [RS:0;9840c0493aab:40395 {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,40395,1733352279076-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,720 INFO [RS:1;9840c0493aab:43247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,720 INFO [RS:0;9840c0493aab:40395 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,720 INFO [RS:1;9840c0493aab:43247 {}] regionserver.Replication(171): 9840c0493aab,43247,1733352279235 started 2024-12-04T22:44:40,720 INFO [RS:0;9840c0493aab:40395 {}] regionserver.Replication(171): 9840c0493aab,40395,1733352279076 started 2024-12-04T22:44:40,752 INFO [RS:1;9840c0493aab:43247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,753 INFO [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(1482): Serving as 9840c0493aab,43247,1733352279235, RpcServer on 9840c0493aab/172.17.0.2:43247, sessionid=0x101a284c1bc0002 2024-12-04T22:44:40,753 INFO [RS:0;9840c0493aab:40395 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:40,754 INFO [RS:0;9840c0493aab:40395 {}] regionserver.HRegionServer(1482): Serving as 9840c0493aab,40395,1733352279076, RpcServer on 9840c0493aab/172.17.0.2:40395, sessionid=0x101a284c1bc0001 2024-12-04T22:44:40,754 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T22:44:40,754 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T22:44:40,755 DEBUG [RS:0;9840c0493aab:40395 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9840c0493aab,40395,1733352279076 2024-12-04T22:44:40,755 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9840c0493aab,40395,1733352279076' 2024-12-04T22:44:40,755 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T22:44:40,756 DEBUG [RS:1;9840c0493aab:43247 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9840c0493aab,43247,1733352279235 2024-12-04T22:44:40,756 WARN [9840c0493aab:41337 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T22:44:40,756 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9840c0493aab,43247,1733352279235' 2024-12-04T22:44:40,756 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T22:44:40,757 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T22:44:40,757 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T22:44:40,757 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T22:44:40,757 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T22:44:40,758 DEBUG [RS:0;9840c0493aab:40395 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9840c0493aab,40395,1733352279076 2024-12-04T22:44:40,758 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9840c0493aab,40395,1733352279076' 2024-12-04T22:44:40,758 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T22:44:40,758 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T22:44:40,758 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T22:44:40,758 DEBUG [RS:1;9840c0493aab:43247 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9840c0493aab,43247,1733352279235 2024-12-04T22:44:40,758 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9840c0493aab,43247,1733352279235' 2024-12-04T22:44:40,758 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T22:44:40,759 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T22:44:40,759 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T22:44:40,760 DEBUG [RS:0;9840c0493aab:40395 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T22:44:40,760 INFO [RS:0;9840c0493aab:40395 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T22:44:40,760 DEBUG [RS:1;9840c0493aab:43247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T22:44:40,760 INFO [RS:1;9840c0493aab:43247 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T22:44:40,760 INFO [RS:0;9840c0493aab:40395 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T22:44:40,760 INFO [RS:1;9840c0493aab:43247 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T22:44:40,865 INFO [RS:0;9840c0493aab:40395 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T22:44:40,865 INFO [RS:1;9840c0493aab:43247 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T22:44:40,867 INFO [RS:0;9840c0493aab:40395 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9840c0493aab%2C40395%2C1733352279076, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,40395,1733352279076, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs, maxLogs=55 2024-12-04T22:44:40,867 INFO [RS:1;9840c0493aab:43247 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9840c0493aab%2C43247%2C1733352279235, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,43247,1733352279235, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs, maxLogs=55 2024-12-04T22:44:40,893 INFO [RS:1;9840c0493aab:43247 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,43247,1733352279235/9840c0493aab%2C43247%2C1733352279235.1733352280869 2024-12-04T22:44:40,893 INFO [RS:0;9840c0493aab:40395 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,40395,1733352279076/9840c0493aab%2C40395%2C1733352279076.1733352280869 2024-12-04T22:44:40,893 DEBUG [RS:1;9840c0493aab:43247 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T22:44:40,893 DEBUG [RS:0;9840c0493aab:40395 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T22:44:41,009 DEBUG [9840c0493aab:41337 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-04T22:44:41,024 DEBUG [9840c0493aab:41337 {}] balancer.BalancerClusterState(204): Hosts are {9840c0493aab=0} racks are {/default-rack=0} 2024-12-04T22:44:41,029 DEBUG [9840c0493aab:41337 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T22:44:41,029 DEBUG [9840c0493aab:41337 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T22:44:41,030 DEBUG [9840c0493aab:41337 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T22:44:41,030 DEBUG [9840c0493aab:41337 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T22:44:41,030 INFO [9840c0493aab:41337 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T22:44:41,030 INFO [9840c0493aab:41337 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T22:44:41,030 DEBUG [9840c0493aab:41337 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T22:44:41,040 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9840c0493aab,43247,1733352279235 2024-12-04T22:44:41,046 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9840c0493aab,43247,1733352279235, state=OPENING 2024-12-04T22:44:41,059 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T22:44:41,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:41,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:41,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:41,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:41,118 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:41,118 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:41,119 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T22:44:41,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9840c0493aab,43247,1733352279235}] 2024-12-04T22:44:41,290 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T22:44:41,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39863, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T22:44:41,301 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T22:44:41,302 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T22:44:41,302 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T22:44:41,303 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9840c0493aab%2C43247%2C1733352279235.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,43247,1733352279235, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs, maxLogs=55 2024-12-04T22:44:41,319 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,43247,1733352279235/9840c0493aab%2C43247%2C1733352279235.meta.1733352281305.meta 2024-12-04T22:44:41,320 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T22:44:41,320 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T22:44:41,322 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T22:44:41,324 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T22:44:41,328 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T22:44:41,332 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T22:44:41,332 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:41,332 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T22:44:41,333 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T22:44:41,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-04T22:44:41,335 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T22:44:41,335 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:41,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:41,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-04T22:44:41,336 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T22:44:41,336 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:41,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:41,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-04T22:44:41,337 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T22:44:41,337 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:41,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:41,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-04T22:44:41,338 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T22:44:41,338 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:41,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:41,338 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T22:44:41,339 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740 2024-12-04T22:44:41,340 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740 2024-12-04T22:44:41,340 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T22:44:41,340 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T22:44:41,340 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T22:44:41,341 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T22:44:41,342 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73060724, jitterRate=0.08868962526321411}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T22:44:41,342 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T22:44:41,344 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733352281333Writing region info on filesystem at 1733352281333Initializing all the Stores at 1733352281333Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352281334 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352281334Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352281334Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352281334Cleaning up temporary data from old regions at 1733352281340 (+6 ms)Running coprocessor post-open hooks at 1733352281342 (+2 ms)Region opened successfully at 1733352281343 (+1 ms) 2024-12-04T22:44:41,348 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733352281283 2024-12-04T22:44:41,357 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T22:44:41,358 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T22:44:41,359 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9840c0493aab,43247,1733352279235 2024-12-04T22:44:41,360 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9840c0493aab,43247,1733352279235, state=OPEN 2024-12-04T22:44:41,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T22:44:41,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T22:44:41,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T22:44:41,370 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:41,370 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:41,370 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:41,370 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9840c0493aab,43247,1733352279235 2024-12-04T22:44:41,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T22:44:41,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9840c0493aab,43247,1733352279235 in 249 msec 2024-12-04T22:44:41,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T22:44:41,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 782 msec 2024-12-04T22:44:41,383 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T22:44:41,383 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T22:44:41,404 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T22:44:41,406 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9840c0493aab,43247,1733352279235, seqNum=-1] 2024-12-04T22:44:41,424 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T22:44:41,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35059, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T22:44:41,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0450 sec 2024-12-04T22:44:41,452 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733352281452, completionTime=-1 2024-12-04T22:44:41,455 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-04T22:44:41,455 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T22:44:41,477 INFO [master/9840c0493aab:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-04T22:44:41,477 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733352341477 2024-12-04T22:44:41,477 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733352401477 2024-12-04T22:44:41,477 INFO [master/9840c0493aab:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-12-04T22:44:41,480 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-04T22:44:41,488 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,41337,1733352277793-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:41,488 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,41337,1733352277793-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:41,488 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,41337,1733352277793-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:41,490 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9840c0493aab:41337, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:41,491 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:41,491 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:41,496 DEBUG [master/9840c0493aab:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T22:44:41,521 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.129sec 2024-12-04T22:44:41,523 INFO [master/9840c0493aab:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T22:44:41,524 INFO [master/9840c0493aab:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T22:44:41,525 INFO [master/9840c0493aab:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T22:44:41,526 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T22:44:41,526 INFO [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T22:44:41,526 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,41337,1733352277793-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T22:44:41,527 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,41337,1733352277793-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T22:44:41,528 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T22:44:41,529 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T22:44:41,529 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,41337,1733352277793-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:41,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109a9e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T22:44:41,611 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T22:44:41,611 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T22:44:41,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9840c0493aab,41337,-1 for getting cluster id 2024-12-04T22:44:41,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T22:44:41,626 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9f1c9b0-bd09-46c1-9c4f-de946f32d48d' 2024-12-04T22:44:41,628 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T22:44:41,628 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9f1c9b0-bd09-46c1-9c4f-de946f32d48d" 2024-12-04T22:44:41,629 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c1bca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T22:44:41,629 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9840c0493aab,41337,-1] 2024-12-04T22:44:41,631 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T22:44:41,633 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:41,634 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T22:44:41,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c35605, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T22:44:41,637 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T22:44:41,643 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9840c0493aab,43247,1733352279235, seqNum=-1] 2024-12-04T22:44:41,643 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T22:44:41,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T22:44:41,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9840c0493aab,41337,1733352277793 2024-12-04T22:44:41,685 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T22:44:41,689 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9840c0493aab,41337,1733352277793 2024-12-04T22:44:41,691 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3a23724d 2024-12-04T22:44:41,692 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T22:44:41,695 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T22:44:41,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41337 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733352281680', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T22:44:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41337 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733352281680 2024-12-04T22:44:41,706 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733352281680 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T22:44:41,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41337 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733352281680" procId is: 4 2024-12-04T22:44:41,708 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:41,710 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733352281680 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T22:44:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41337 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T22:44:41,724 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733352281680-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 32ca1fed4af4ed59e194130ad78c2419, NAME => 'testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733352281680', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b 2024-12-04T22:44:41,730 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733352281680-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:41,730 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733352281680-pool-0 {}] regionserver.HRegion(1722): Closing 32ca1fed4af4ed59e194130ad78c2419, disabling compactions & flushes 2024-12-04T22:44:41,730 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733352281680-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:41,730 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733352281680-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:41,730 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733352281680-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. after waiting 0 ms 2024-12-04T22:44:41,730 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733352281680-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:41,730 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733352281680-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:41,730 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733352281680-pool-0 {}] regionserver.HRegion(1676): Region close journal for 32ca1fed4af4ed59e194130ad78c2419: Waiting for close lock at 1733352281730Disabling compacts and flushes for region at 1733352281730Disabling writes for close at 1733352281730Writing region close event to WAL at 1733352281730Closed at 1733352281730 2024-12-04T22:44:41,731 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733352281680 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T22:44:41,736 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733352281732"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733352281732"}]},"ts":"1733352281732"} 2024-12-04T22:44:41,739 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T22:44:41,741 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733352281680 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T22:44:41,743 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733352281680","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733352281741"}]},"ts":"1733352281741"} 2024-12-04T22:44:41,746 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733352281680, state=ENABLING in hbase:meta 2024-12-04T22:44:41,746 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {9840c0493aab=0} racks are {/default-rack=0} 2024-12-04T22:44:41,747 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T22:44:41,747 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T22:44:41,747 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T22:44:41,747 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T22:44:41,747 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T22:44:41,747 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T22:44:41,747 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T22:44:41,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733352281680, region=32ca1fed4af4ed59e194130ad78c2419, ASSIGN}] 2024-12-04T22:44:41,750 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733352281680, region=32ca1fed4af4ed59e194130ad78c2419, ASSIGN 2024-12-04T22:44:41,751 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733352281680, region=32ca1fed4af4ed59e194130ad78c2419, ASSIGN; state=OFFLINE, location=9840c0493aab,40395,1733352279076; forceNewPlan=false, retain=false 2024-12-04T22:44:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41337 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T22:44:41,904 INFO [9840c0493aab:41337 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T22:44:41,905 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=32ca1fed4af4ed59e194130ad78c2419, regionState=OPENING, regionLocation=9840c0493aab,40395,1733352279076 2024-12-04T22:44:41,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733352281680, region=32ca1fed4af4ed59e194130ad78c2419, ASSIGN because future has completed 2024-12-04T22:44:41,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32ca1fed4af4ed59e194130ad78c2419, server=9840c0493aab,40395,1733352279076}] 2024-12-04T22:44:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41337 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T22:44:42,073 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T22:44:42,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34613, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T22:44:42,080 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:42,081 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 32ca1fed4af4ed59e194130ad78c2419, NAME => 'testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419.', STARTKEY => '', ENDKEY => ''} 2024-12-04T22:44:42,083 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733352281680 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:42,083 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:42,084 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:42,084 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:42,089 INFO [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:42,113 INFO [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32ca1fed4af4ed59e194130ad78c2419 columnFamilyName family 2024-12-04T22:44:42,113 DEBUG [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:42,114 INFO [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] regionserver.HStore(327): Store=32ca1fed4af4ed59e194130ad78c2419/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T22:44:42,115 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:42,116 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:42,116 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:42,117 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:42,117 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:42,118 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:42,129 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T22:44:42,130 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 32ca1fed4af4ed59e194130ad78c2419; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61230174, jitterRate=-0.08759930729866028}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T22:44:42,131 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:42,132 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 32ca1fed4af4ed59e194130ad78c2419: Running coprocessor pre-open hook at 1733352282084Writing region info on filesystem at 1733352282084Initializing all the Stores at 1733352282085 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352282085Cleaning up temporary data from old regions at 1733352282117 (+32 ms)Running coprocessor post-open hooks at 1733352282131 (+14 ms)Region opened successfully at 1733352282132 (+1 ms) 2024-12-04T22:44:42,134 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419., pid=6, masterSystemTime=1733352282072 2024-12-04T22:44:42,138 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:42,138 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:42,139 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=32ca1fed4af4ed59e194130ad78c2419, regionState=OPEN, openSeqNum=2, regionLocation=9840c0493aab,40395,1733352279076 2024-12-04T22:44:42,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32ca1fed4af4ed59e194130ad78c2419, server=9840c0493aab,40395,1733352279076 because future has completed 2024-12-04T22:44:42,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T22:44:42,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 32ca1fed4af4ed59e194130ad78c2419, server=9840c0493aab,40395,1733352279076 in 231 msec 2024-12-04T22:44:42,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T22:44:42,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733352281680, region=32ca1fed4af4ed59e194130ad78c2419, ASSIGN in 404 msec 2024-12-04T22:44:42,161 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733352281680 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T22:44:42,161 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733352281680","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733352282161"}]},"ts":"1733352282161"} 2024-12-04T22:44:42,166 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733352281680, state=ENABLED in hbase:meta 2024-12-04T22:44:42,167 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733352281680 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T22:44:42,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733352281680 in 465 msec 2024-12-04T22:44:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41337 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T22:44:42,342 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733352281680 get assigned. Timeout = 60000ms 2024-12-04T22:44:42,342 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733352281680 completed 2024-12-04T22:44:42,346 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T22:44:42,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733352281680 assigned to meta. Checking AM states. 2024-12-04T22:44:42,355 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T22:44:42,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733352281680 assigned. 2024-12-04T22:44:42,368 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733352281680', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419., hostname=9840c0493aab,40395,1733352279076, seqNum=2] 2024-12-04T22:44:42,370 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T22:44:42,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T22:44:42,394 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 32ca1fed4af4ed59e194130ad78c2419 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-04T22:44:42,444 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 089d5c0127cf4faabca3b894867c9b2f 2024-12-04T22:44:42,512 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/.tmp/family/089d5c0127cf4faabca3b894867c9b2f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-04T22:44:42,519 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/.tmp/family/089d5c0127cf4faabca3b894867c9b2f 2024-12-04T22:44:42,569 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/.tmp/family/089d5c0127cf4faabca3b894867c9b2f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f 2024-12-04T22:44:42,571 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f, delay=903 ms 2024-12-04T22:44:42,573 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f, entries=2, sequenceid=6, filesize=4.9 K 2024-12-04T22:44:42,581 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 32ca1fed4af4ed59e194130ad78c2419 in 187ms, sequenceid=6, compaction requested=false 2024-12-04T22:44:42,582 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733352281680' 2024-12-04T22:44:42,585 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 32ca1fed4af4ed59e194130ad78c2419: 2024-12-04T22:44:43,477 DEBUG [hfile-prefetch-1733352282571 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 089d5c0127cf4faabca3b894867c9b2f_0. 2024-12-04T22:44:43,482 DEBUG [hfile-prefetch-1733352282571 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 089d5c0127cf4faabca3b894867c9b2f_117. 2024-12-04T22:44:43,483 DEBUG [hfile-prefetch-1733352282571 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f, with total blocks 2, and data blocks 1 2024-12-04T22:44:43,483 DEBUG [hfile-prefetch-1733352282571 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-04T22:44:43,484 DEBUG [hfile-prefetch-1733352282571 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-04T22:44:43,484 DEBUG [hfile-prefetch-1733352282571 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f. Will sleep 100ms and try the verification again. 2024-12-04T22:44:43,584 DEBUG [hfile-prefetch-1733352282571 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f, with total blocks 2, and data blocks 1 2024-12-04T22:44:43,584 DEBUG [hfile-prefetch-1733352282571 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-04T22:44:43,585 DEBUG [hfile-prefetch-1733352282571 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-04T22:44:43,585 DEBUG [hfile-prefetch-1733352282571 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f. Will sleep 100ms and try the verification again. 2024-12-04T22:44:43,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41337 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=32ca1fed4af4ed59e194130ad78c2419, source=9840c0493aab,40395,1733352279076, destination=9840c0493aab,43247,1733352279235, warming up region on 9840c0493aab,43247,1733352279235 2024-12-04T22:44:43,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41337 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=32ca1fed4af4ed59e194130ad78c2419, source=9840c0493aab,40395,1733352279076, destination=9840c0493aab,43247,1733352279235, running balancer 2024-12-04T22:44:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41337 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733352281680, region=32ca1fed4af4ed59e194130ad78c2419, REOPEN/MOVE 2024-12-04T22:44:43,601 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733352281680, region=32ca1fed4af4ed59e194130ad78c2419, REOPEN/MOVE 2024-12-04T22:44:43,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43247 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43247 {}] regionserver.HRegion(7855): Warmup {ENCODED => 32ca1fed4af4ed59e194130ad78c2419, NAME => 'testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419.', STARTKEY => '', ENDKEY => ''} 2024-12-04T22:44:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43247 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:43,603 INFO [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:43,603 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=32ca1fed4af4ed59e194130ad78c2419, regionState=CLOSING, regionLocation=9840c0493aab,40395,1733352279076 2024-12-04T22:44:43,604 INFO [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32ca1fed4af4ed59e194130ad78c2419 columnFamilyName family 2024-12-04T22:44:43,604 DEBUG [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:43,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733352281680, region=32ca1fed4af4ed59e194130ad78c2419, REOPEN/MOVE because future has completed 2024-12-04T22:44:43,607 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-04T22:44:43,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 32ca1fed4af4ed59e194130ad78c2419, server=9840c0493aab,40395,1733352279076}] 2024-12-04T22:44:43,608 DEBUG [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f 2024-12-04T22:44:43,608 INFO [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] regionserver.HStore(327): Store=32ca1fed4af4ed59e194130ad78c2419/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T22:44:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43247 {}] regionserver.HRegion(1722): Closing 32ca1fed4af4ed59e194130ad78c2419, disabling compactions & flushes 2024-12-04T22:44:43,608 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43247 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43247 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43247 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. after waiting 0 ms 2024-12-04T22:44:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43247 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:43,612 DEBUG [StoreFileCloser-32ca1fed4af4ed59e194130ad78c2419-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f 2024-12-04T22:44:43,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43247 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43247 {}] regionserver.HRegion(1676): Region close journal for 32ca1fed4af4ed59e194130ad78c2419: Waiting for close lock at 1733352283608Disabling compacts and flushes for region at 1733352283608Disabling writes for close at 1733352283608Writing region close event to WAL at 1733352283613 (+5 ms)Closed at 1733352283613 2024-12-04T22:44:43,614 WARN [hfile-prefetch-1733352282571 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 10 more 2024-12-04T22:44:43,615 DEBUG [hfile-prefetch-1733352282571 {}] hfile.PrefetchExecutor(121): Prefetch completed for 089d5c0127cf4faabca3b894867c9b2f 2024-12-04T22:44:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41337 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-04T22:44:43,678 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-04T22:44:43,678 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-04T22:44:43,679 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T22:44:43,679 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T22:44:43,682 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-04T22:44:43,682 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-04T22:44:43,682 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T22:44:43,682 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T22:44:43,692 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T22:44:43,692 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733352283678 2024-12-04T22:44:43,692 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-04T22:44:43,695 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-04T22:44:43,695 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733352283682 2024-12-04T22:44:43,695 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-04T22:44:43,768 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:43,769 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-04T22:44:43,770 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 32ca1fed4af4ed59e194130ad78c2419, disabling compactions & flushes 2024-12-04T22:44:43,770 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:43,770 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:43,770 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. after waiting 0 ms 2024-12-04T22:44:43,770 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:43,773 DEBUG [StoreFileCloser-32ca1fed4af4ed59e194130ad78c2419-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 089d5c0127cf4faabca3b894867c9b2f, starting offset: 0, end offset: 9223372036854775807 2024-12-04T22:44:43,776 DEBUG [StoreFileCloser-32ca1fed4af4ed59e194130ad78c2419-family-1 {}] bucket.BucketCache(856): removed key 089d5c0127cf4faabca3b894867c9b2f_0 from back map with offset lock 2094080 in the evict process 2024-12-04T22:44:43,783 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-04T22:44:43,784 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:43,784 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 32ca1fed4af4ed59e194130ad78c2419: Waiting for close lock at 1733352283769Running coprocessor pre-close hooks at 1733352283770 (+1 ms)Disabling compacts and flushes for region at 1733352283770Disabling writes for close at 1733352283770Writing region close event to WAL at 1733352283777 (+7 ms)Running coprocessor post-close hooks at 1733352283783 (+6 ms)Closed at 1733352283784 (+1 ms) 2024-12-04T22:44:43,784 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 32ca1fed4af4ed59e194130ad78c2419 move to 9840c0493aab,43247,1733352279235 record at close sequenceid=6 2024-12-04T22:44:43,787 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:43,788 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=32ca1fed4af4ed59e194130ad78c2419, regionState=CLOSED 2024-12-04T22:44:43,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 32ca1fed4af4ed59e194130ad78c2419, server=9840c0493aab,40395,1733352279076 because future has completed 2024-12-04T22:44:43,793 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-04T22:44:43,793 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-04T22:44:43,793 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T22:44:43,793 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T22:44:43,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T22:44:43,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 32ca1fed4af4ed59e194130ad78c2419, server=9840c0493aab,40395,1733352279076 in 184 msec 2024-12-04T22:44:43,794 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733352281680, region=32ca1fed4af4ed59e194130ad78c2419, REOPEN/MOVE; state=CLOSED, location=9840c0493aab,43247,1733352279235; forceNewPlan=false, retain=false 2024-12-04T22:44:43,796 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T22:44:43,796 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733352283793 2024-12-04T22:44:43,796 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-04T22:44:43,945 INFO [9840c0493aab:41337 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T22:44:43,945 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=32ca1fed4af4ed59e194130ad78c2419, regionState=OPENING, regionLocation=9840c0493aab,43247,1733352279235 2024-12-04T22:44:43,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733352281680, region=32ca1fed4af4ed59e194130ad78c2419, REOPEN/MOVE because future has completed 2024-12-04T22:44:43,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32ca1fed4af4ed59e194130ad78c2419, server=9840c0493aab,43247,1733352279235}] 2024-12-04T22:44:44,106 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:44,106 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 32ca1fed4af4ed59e194130ad78c2419, NAME => 'testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419.', STARTKEY => '', ENDKEY => ''} 2024-12-04T22:44:44,107 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733352281680 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,107 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:44,107 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,107 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,112 INFO [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,114 INFO [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32ca1fed4af4ed59e194130ad78c2419 columnFamilyName family 2024-12-04T22:44:44,114 DEBUG [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:44,118 DEBUG [StoreFileOpener-32ca1fed4af4ed59e194130ad78c2419-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f, delay=963 ms 2024-12-04T22:44:44,119 DEBUG [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f 2024-12-04T22:44:44,120 INFO [StoreOpener-32ca1fed4af4ed59e194130ad78c2419-1 {}] regionserver.HStore(327): Store=32ca1fed4af4ed59e194130ad78c2419/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T22:44:44,120 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,121 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,122 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,122 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,122 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,129 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,130 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 32ca1fed4af4ed59e194130ad78c2419; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73254781, jitterRate=0.0915812999010086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T22:44:44,130 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,132 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 32ca1fed4af4ed59e194130ad78c2419: Running coprocessor pre-open hook at 1733352284108Writing region info on filesystem at 1733352284108Initializing all the Stores at 1733352284110 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352284110Cleaning up temporary data from old regions at 1733352284122 (+12 ms)Running coprocessor post-open hooks at 1733352284131 (+9 ms)Region opened successfully at 1733352284132 (+1 ms) 2024-12-04T22:44:44,133 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419., pid=9, masterSystemTime=1733352284102 2024-12-04T22:44:44,138 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:44,138 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:44,139 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=32ca1fed4af4ed59e194130ad78c2419, regionState=OPEN, openSeqNum=10, regionLocation=9840c0493aab,43247,1733352279235 2024-12-04T22:44:44,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32ca1fed4af4ed59e194130ad78c2419, server=9840c0493aab,43247,1733352279235 because future has completed 2024-12-04T22:44:44,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-04T22:44:44,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 32ca1fed4af4ed59e194130ad78c2419, server=9840c0493aab,43247,1733352279235 in 196 msec 2024-12-04T22:44:44,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733352281680, region=32ca1fed4af4ed59e194130ad78c2419, REOPEN/MOVE in 551 msec 2024-12-04T22:44:44,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T22:44:44,205 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T22:44:44,205 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T22:44:44,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:44,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:44,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T22:44:44,213 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T22:44:44,213 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=632300009, stopped=false 2024-12-04T22:44:44,214 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9840c0493aab,41337,1733352277793 2024-12-04T22:44:44,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T22:44:44,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T22:44:44,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:44,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T22:44:44,228 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T22:44:44,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:44,229 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T22:44:44,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:44,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T22:44:44,229 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T22:44:44,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T22:44:44,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:44,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T22:44:44,230 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9840c0493aab,40395,1733352279076' ***** 2024-12-04T22:44:44,230 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T22:44:44,230 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9840c0493aab,43247,1733352279235' ***** 2024-12-04T22:44:44,230 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T22:44:44,230 INFO [RS:0;9840c0493aab:40395 {}] bucket.BucketCache(1738): Disabling cache 2024-12-04T22:44:44,230 INFO [RS:0;9840c0493aab:40395 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/bucket.persistence 2024-12-04T22:44:44,230 INFO [RS:1;9840c0493aab:43247 {}] bucket.BucketCache(1738): Disabling cache 2024-12-04T22:44:44,230 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-04T22:44:44,230 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-04T22:44:44,230 INFO [RS:1;9840c0493aab:43247 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/bucket.persistence 2024-12-04T22:44:44,231 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-04T22:44:44,231 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-04T22:44:44,231 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-04T22:44:44,231 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-04T22:44:44,231 INFO [RS:0;9840c0493aab:40395 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-04T22:44:44,231 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-04T22:44:44,232 INFO [RS:1;9840c0493aab:43247 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-04T22:44:44,232 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-04T22:44:44,241 DEBUG [RS:0;9840c0493aab:40395 {}] bucket.BucketCache(1487): Thread RS:0;9840c0493aab:40395 started persisting bucket cache to file 2024-12-04T22:44:44,242 DEBUG [RS:0;9840c0493aab:40395 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T22:44:44,242 DEBUG [RS:0;9840c0493aab:40395 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T22:44:44,242 DEBUG [RS:1;9840c0493aab:43247 {}] bucket.BucketCache(1487): Thread RS:1;9840c0493aab:43247 started persisting bucket cache to file 2024-12-04T22:44:44,242 DEBUG [RS:1;9840c0493aab:43247 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T22:44:44,242 DEBUG [RS:1;9840c0493aab:43247 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T22:44:44,249 DEBUG [RS:1;9840c0493aab:43247 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T22:44:44,249 DEBUG [RS:0;9840c0493aab:40395 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T22:44:44,249 DEBUG [RS:1;9840c0493aab:43247 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733352284242 2024-12-04T22:44:44,249 DEBUG [RS:0;9840c0493aab:40395 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733352284241 2024-12-04T22:44:44,249 DEBUG [RS:1;9840c0493aab:43247 {}] bucket.BucketCache(1509): Thread RS:1;9840c0493aab:43247 finished persisting bucket cache to file, renaming 2024-12-04T22:44:44,249 DEBUG [RS:0;9840c0493aab:40395 {}] bucket.BucketCache(1509): Thread RS:0;9840c0493aab:40395 finished persisting bucket cache to file, renaming 2024-12-04T22:44:44,249 INFO [RS:1;9840c0493aab:43247 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T22:44:44,249 INFO [RS:0;9840c0493aab:40395 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T22:44:44,250 INFO [RS:0;9840c0493aab:40395 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T22:44:44,250 INFO [RS:1;9840c0493aab:43247 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T22:44:44,250 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T22:44:44,250 INFO [RS:0;9840c0493aab:40395 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T22:44:44,250 INFO [RS:1;9840c0493aab:43247 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T22:44:44,250 INFO [RS:0;9840c0493aab:40395 {}] regionserver.HRegionServer(959): stopping server 9840c0493aab,40395,1733352279076 2024-12-04T22:44:44,250 INFO [RS:0;9840c0493aab:40395 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T22:44:44,250 INFO [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(3091): Received CLOSE for 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,250 INFO [RS:0;9840c0493aab:40395 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9840c0493aab:40395. 2024-12-04T22:44:44,250 DEBUG [RS:0;9840c0493aab:40395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T22:44:44,250 DEBUG [RS:0;9840c0493aab:40395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:44,250 INFO [RS:0;9840c0493aab:40395 {}] regionserver.HRegionServer(976): stopping server 9840c0493aab,40395,1733352279076; all regions closed. 2024-12-04T22:44:44,251 INFO [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(959): stopping server 9840c0493aab,43247,1733352279235 2024-12-04T22:44:44,251 INFO [RS:1;9840c0493aab:43247 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T22:44:44,251 INFO [RS:1;9840c0493aab:43247 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9840c0493aab:43247. 2024-12-04T22:44:44,251 DEBUG [RS:1;9840c0493aab:43247 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T22:44:44,251 DEBUG [RS:1;9840c0493aab:43247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:44,251 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T22:44:44,251 INFO [RS:1;9840c0493aab:43247 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T22:44:44,251 INFO [RS:1;9840c0493aab:43247 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T22:44:44,251 INFO [RS:1;9840c0493aab:43247 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T22:44:44,251 INFO [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T22:44:44,252 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-04T22:44:44,253 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 32ca1fed4af4ed59e194130ad78c2419, disabling compactions & flushes 2024-12-04T22:44:44,253 INFO [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T22:44:44,253 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:44,253 DEBUG [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(1325): Online Regions={32ca1fed4af4ed59e194130ad78c2419=testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T22:44:44,253 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:44,253 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-04T22:44:44,253 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. after waiting 0 ms 2024-12-04T22:44:44,253 DEBUG [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 32ca1fed4af4ed59e194130ad78c2419 2024-12-04T22:44:44,253 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:44,255 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T22:44:44,255 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T22:44:44,255 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T22:44:44,255 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T22:44:44,255 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T22:44:44,256 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-04T22:44:44,259 DEBUG [RS:0;9840c0493aab:40395 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs 2024-12-04T22:44:44,259 INFO [RS:0;9840c0493aab:40395 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9840c0493aab%2C40395%2C1733352279076:(num 1733352280869) 2024-12-04T22:44:44,260 DEBUG [RS:0;9840c0493aab:40395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:44,260 INFO [RS:0;9840c0493aab:40395 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T22:44:44,260 INFO [RS:0;9840c0493aab:40395 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T22:44:44,260 INFO [RS:0;9840c0493aab:40395 {}] hbase.ChoreService(370): Chore service for: regionserver/9840c0493aab:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T22:44:44,260 INFO [RS:0;9840c0493aab:40395 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T22:44:44,260 INFO [regionserver/9840c0493aab:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T22:44:44,260 INFO [RS:0;9840c0493aab:40395 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T22:44:44,261 INFO [RS:0;9840c0493aab:40395 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T22:44:44,261 INFO [RS:0;9840c0493aab:40395 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T22:44:44,261 INFO [RS:0;9840c0493aab:40395 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40395 2024-12-04T22:44:44,264 DEBUG [StoreFileCloser-32ca1fed4af4ed59e194130ad78c2419-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/family/089d5c0127cf4faabca3b894867c9b2f 2024-12-04T22:44:44,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9840c0493aab,40395,1733352279076 2024-12-04T22:44:44,274 INFO [RS:0;9840c0493aab:40395 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T22:44:44,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T22:44:44,275 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnRegionMove1733352281680/32ca1fed4af4ed59e194130ad78c2419/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-04T22:44:44,275 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:44,275 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 32ca1fed4af4ed59e194130ad78c2419: Waiting for close lock at 1733352284252Running coprocessor pre-close hooks at 1733352284253 (+1 ms)Disabling compacts and flushes for region at 1733352284253Disabling writes for close at 1733352284253Writing region close event to WAL at 1733352284265 (+12 ms)Running coprocessor post-close hooks at 1733352284275 (+10 ms)Closed at 1733352284275 2024-12-04T22:44:44,276 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419. 2024-12-04T22:44:44,277 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9840c0493aab,40395,1733352279076] 2024-12-04T22:44:44,290 INFO [regionserver/9840c0493aab:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T22:44:44,290 INFO [regionserver/9840c0493aab:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T22:44:44,295 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9840c0493aab,40395,1733352279076 already deleted, retry=false 2024-12-04T22:44:44,295 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9840c0493aab,40395,1733352279076 expired; onlineServers=1 2024-12-04T22:44:44,301 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a3e1cec973a549d19bff2e59e3fd4ed2 2024-12-04T22:44:44,325 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/info/a3e1cec973a549d19bff2e59e3fd4ed2 is 205, key is testBlockEvictionOnRegionMove1733352281680,,1733352281696.32ca1fed4af4ed59e194130ad78c2419./info:regioninfo/1733352284138/Put/seqid=0 2024-12-04T22:44:44,327 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/info/a3e1cec973a549d19bff2e59e3fd4ed2 2024-12-04T22:44:44,357 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f362cd26bb4146d0b8c5ca856297ebf1 2024-12-04T22:44:44,374 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/ns/f362cd26bb4146d0b8c5ca856297ebf1 is 43, key is default/ns:d/1733352281431/Put/seqid=0 2024-12-04T22:44:44,375 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/ns/f362cd26bb4146d0b8c5ca856297ebf1 2024-12-04T22:44:44,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:44,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40395-0x101a284c1bc0001, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:44,387 INFO [RS:0;9840c0493aab:40395 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T22:44:44,387 INFO [RS:0;9840c0493aab:40395 {}] regionserver.HRegionServer(1031): Exiting; stopping=9840c0493aab,40395,1733352279076; zookeeper connection closed. 2024-12-04T22:44:44,390 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@125b8ed1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@125b8ed1 2024-12-04T22:44:44,405 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bc707f7fcee347b4a2f5ecf3b6c28486 2024-12-04T22:44:44,421 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/table/bc707f7fcee347b4a2f5ecf3b6c28486 is 78, key is testBlockEvictionOnRegionMove1733352281680/table:state/1733352282161/Put/seqid=0 2024-12-04T22:44:44,422 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/table/bc707f7fcee347b4a2f5ecf3b6c28486 2024-12-04T22:44:44,426 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/info/a3e1cec973a549d19bff2e59e3fd4ed2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/a3e1cec973a549d19bff2e59e3fd4ed2 2024-12-04T22:44:44,427 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/a3e1cec973a549d19bff2e59e3fd4ed2, delay=991 ms 2024-12-04T22:44:44,429 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/a3e1cec973a549d19bff2e59e3fd4ed2, entries=15, sequenceid=15, filesize=7.6 K 2024-12-04T22:44:44,430 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/ns/f362cd26bb4146d0b8c5ca856297ebf1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/ns/f362cd26bb4146d0b8c5ca856297ebf1 2024-12-04T22:44:44,434 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/ns/f362cd26bb4146d0b8c5ca856297ebf1, delay=947 ms 2024-12-04T22:44:44,437 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/ns/f362cd26bb4146d0b8c5ca856297ebf1, entries=2, sequenceid=15, filesize=5.0 K 2024-12-04T22:44:44,437 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/table/bc707f7fcee347b4a2f5ecf3b6c28486 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/table/bc707f7fcee347b4a2f5ecf3b6c28486 2024-12-04T22:44:44,440 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/table/bc707f7fcee347b4a2f5ecf3b6c28486, delay=921 ms 2024-12-04T22:44:44,442 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/table/bc707f7fcee347b4a2f5ecf3b6c28486, entries=2, sequenceid=15, filesize=5.3 K 2024-12-04T22:44:44,444 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 188ms, sequenceid=15, compaction requested=false 2024-12-04T22:44:44,444 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T22:44:44,448 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/a3e1cec973a549d19bff2e59e3fd4ed2 2024-12-04T22:44:44,451 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/ns/f362cd26bb4146d0b8c5ca856297ebf1 2024-12-04T22:44:44,452 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/table/bc707f7fcee347b4a2f5ecf3b6c28486 2024-12-04T22:44:44,454 DEBUG [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T22:44:44,461 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-04T22:44:44,462 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T22:44:44,462 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T22:44:44,463 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733352284253Running coprocessor pre-close hooks at 1733352284253Disabling compacts and flushes for region at 1733352284255 (+2 ms)Disabling writes for close at 1733352284255Obtaining lock to block concurrent updates at 1733352284256 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733352284256Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733352284257 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733352284257Flushing 1588230740/info: creating writer at 1733352284258 (+1 ms)Flushing 1588230740/info: appending metadata at 1733352284323 (+65 ms)Flushing 1588230740/info: closing flushed file at 1733352284324 (+1 ms)Flushing 1588230740/ns: creating writer at 1733352284331 (+7 ms)Flushing 1588230740/ns: appending metadata at 1733352284373 (+42 ms)Flushing 1588230740/ns: closing flushed file at 1733352284373Flushing 1588230740/table: creating writer at 1733352284379 (+6 ms)Flushing 1588230740/table: appending metadata at 1733352284420 (+41 ms)Flushing 1588230740/table: closing flushed file at 1733352284420Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fcfbc2d: reopening flushed file at 1733352284425 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5be1db6a: reopening flushed file at 1733352284430 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@392f5e4f: reopening flushed file at 1733352284437 (+7 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 188ms, sequenceid=15, compaction requested=false at 1733352284444 (+7 ms)Writing region close event to WAL at 1733352284453 (+9 ms)Running coprocessor post-close hooks at 1733352284461 (+8 ms)Closed at 1733352284462 (+1 ms) 2024-12-04T22:44:44,463 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T22:44:44,654 INFO [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(976): stopping server 9840c0493aab,43247,1733352279235; all regions closed. 2024-12-04T22:44:44,658 DEBUG [RS:1;9840c0493aab:43247 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs 2024-12-04T22:44:44,658 INFO [RS:1;9840c0493aab:43247 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9840c0493aab%2C43247%2C1733352279235.meta:.meta(num 1733352281305) 2024-12-04T22:44:44,660 DEBUG [RS:1;9840c0493aab:43247 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs 2024-12-04T22:44:44,660 INFO [RS:1;9840c0493aab:43247 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9840c0493aab%2C43247%2C1733352279235:(num 1733352280869) 2024-12-04T22:44:44,660 DEBUG [RS:1;9840c0493aab:43247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:44,661 INFO [RS:1;9840c0493aab:43247 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T22:44:44,661 INFO [RS:1;9840c0493aab:43247 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T22:44:44,661 INFO [RS:1;9840c0493aab:43247 {}] hbase.ChoreService(370): Chore service for: regionserver/9840c0493aab:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T22:44:44,661 INFO [RS:1;9840c0493aab:43247 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T22:44:44,661 INFO [regionserver/9840c0493aab:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T22:44:44,661 INFO [RS:1;9840c0493aab:43247 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43247 2024-12-04T22:44:44,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9840c0493aab,43247,1733352279235 2024-12-04T22:44:44,674 INFO [RS:1;9840c0493aab:43247 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T22:44:44,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T22:44:44,674 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007feea0477b20@51963128 rejected from java.util.concurrent.ThreadPoolExecutor@1494b57e[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 14] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-04T22:44:44,685 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9840c0493aab,43247,1733352279235] 2024-12-04T22:44:44,695 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9840c0493aab,43247,1733352279235 already deleted, retry=false 2024-12-04T22:44:44,695 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9840c0493aab,43247,1733352279235 expired; onlineServers=0 2024-12-04T22:44:44,696 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9840c0493aab,41337,1733352277793' ***** 2024-12-04T22:44:44,696 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T22:44:44,696 INFO [M:0;9840c0493aab:41337 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T22:44:44,696 INFO [M:0;9840c0493aab:41337 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T22:44:44,696 DEBUG [M:0;9840c0493aab:41337 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T22:44:44,696 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T22:44:44,697 DEBUG [M:0;9840c0493aab:41337 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T22:44:44,697 DEBUG [master/9840c0493aab:0:becomeActiveMaster-HFileCleaner.large.0-1733352280520 {}] cleaner.HFileCleaner(306): Exit Thread[master/9840c0493aab:0:becomeActiveMaster-HFileCleaner.large.0-1733352280520,5,FailOnTimeoutGroup] 2024-12-04T22:44:44,697 DEBUG [master/9840c0493aab:0:becomeActiveMaster-HFileCleaner.small.0-1733352280521 {}] cleaner.HFileCleaner(306): Exit Thread[master/9840c0493aab:0:becomeActiveMaster-HFileCleaner.small.0-1733352280521,5,FailOnTimeoutGroup] 2024-12-04T22:44:44,697 INFO [M:0;9840c0493aab:41337 {}] hbase.ChoreService(370): Chore service for: master/9840c0493aab:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T22:44:44,697 INFO [M:0;9840c0493aab:41337 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T22:44:44,697 DEBUG [M:0;9840c0493aab:41337 {}] master.HMaster(1795): Stopping service threads 2024-12-04T22:44:44,697 INFO [M:0;9840c0493aab:41337 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T22:44:44,698 INFO [M:0;9840c0493aab:41337 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T22:44:44,698 INFO [M:0;9840c0493aab:41337 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T22:44:44,698 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T22:44:44,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T22:44:44,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:44,706 DEBUG [M:0;9840c0493aab:41337 {}] zookeeper.ZKUtil(347): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T22:44:44,706 WARN [M:0;9840c0493aab:41337 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T22:44:44,707 INFO [M:0;9840c0493aab:41337 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/.lastflushedseqids 2024-12-04T22:44:44,734 INFO [M:0;9840c0493aab:41337 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T22:44:44,735 INFO [M:0;9840c0493aab:41337 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T22:44:44,735 DEBUG [M:0;9840c0493aab:41337 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T22:44:44,735 INFO [M:0;9840c0493aab:41337 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:44,735 DEBUG [M:0;9840c0493aab:41337 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:44,735 DEBUG [M:0;9840c0493aab:41337 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T22:44:44,735 DEBUG [M:0;9840c0493aab:41337 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:44,735 INFO [M:0;9840c0493aab:41337 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-12-04T22:44:44,745 DEBUG [M:0;9840c0493aab:41337 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d898b6d36e13446685197c5e9f099eaa 2024-12-04T22:44:44,763 DEBUG [M:0;9840c0493aab:41337 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d898b6d36e13446685197c5e9f099eaa is 82, key is hbase:meta,,1/info:regioninfo/1733352281358/Put/seqid=0 2024-12-04T22:44:44,765 INFO [M:0;9840c0493aab:41337 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d898b6d36e13446685197c5e9f099eaa 2024-12-04T22:44:44,776 DEBUG [M:0;9840c0493aab:41337 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d019534d0a4b4886b4745fd8453b817d 2024-12-04T22:44:44,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:44,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43247-0x101a284c1bc0002, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:44,785 INFO [RS:1;9840c0493aab:43247 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T22:44:44,785 INFO [RS:1;9840c0493aab:43247 {}] regionserver.HRegionServer(1031): Exiting; stopping=9840c0493aab,43247,1733352279235; zookeeper connection closed. 2024-12-04T22:44:44,785 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75be3407 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75be3407 2024-12-04T22:44:44,786 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-04T22:44:44,793 DEBUG [M:0;9840c0493aab:41337 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d019534d0a4b4886b4745fd8453b817d is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733352282170/Put/seqid=0 2024-12-04T22:44:44,794 INFO [M:0;9840c0493aab:41337 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d019534d0a4b4886b4745fd8453b817d 2024-12-04T22:44:44,808 DEBUG [M:0;9840c0493aab:41337 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9e3674988e1e4484a00f2da5e05f404b 2024-12-04T22:44:44,827 DEBUG [M:0;9840c0493aab:41337 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e3674988e1e4484a00f2da5e05f404b is 69, key is 9840c0493aab,40395,1733352279076/rs:state/1733352280558/Put/seqid=0 2024-12-04T22:44:44,828 INFO [M:0;9840c0493aab:41337 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e3674988e1e4484a00f2da5e05f404b 2024-12-04T22:44:44,832 DEBUG [M:0;9840c0493aab:41337 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d898b6d36e13446685197c5e9f099eaa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d898b6d36e13446685197c5e9f099eaa 2024-12-04T22:44:44,835 INFO [M:0;9840c0493aab:41337 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d898b6d36e13446685197c5e9f099eaa, entries=8, sequenceid=84, filesize=5.5 K 2024-12-04T22:44:44,836 DEBUG [M:0;9840c0493aab:41337 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d019534d0a4b4886b4745fd8453b817d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d019534d0a4b4886b4745fd8453b817d 2024-12-04T22:44:44,839 INFO [M:0;9840c0493aab:41337 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d019534d0a4b4886b4745fd8453b817d, entries=9, sequenceid=84, filesize=6.5 K 2024-12-04T22:44:44,840 DEBUG [M:0;9840c0493aab:41337 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e3674988e1e4484a00f2da5e05f404b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9e3674988e1e4484a00f2da5e05f404b 2024-12-04T22:44:44,842 INFO [M:0;9840c0493aab:41337 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9e3674988e1e4484a00f2da5e05f404b, entries=2, sequenceid=84, filesize=5.1 K 2024-12-04T22:44:44,844 INFO [M:0;9840c0493aab:41337 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=84, compaction requested=false 2024-12-04T22:44:44,859 INFO [M:0;9840c0493aab:41337 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:44,859 DEBUG [M:0;9840c0493aab:41337 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733352284735Disabling compacts and flushes for region at 1733352284735Disabling writes for close at 1733352284735Obtaining lock to block concurrent updates at 1733352284735Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733352284735Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733352284736 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733352284736Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733352284736Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733352284763 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733352284763Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733352284768 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733352284792 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733352284792Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733352284797 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733352284827 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733352284827Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67f69d6a: reopening flushed file at 1733352284832 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78a55e7f: reopening flushed file at 1733352284835 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@653bb434: reopening flushed file at 1733352284839 (+4 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=84, compaction requested=false at 1733352284844 (+5 ms)Writing region close event to WAL at 1733352284859 (+15 ms)Closed at 1733352284859 2024-12-04T22:44:44,865 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T22:44:44,865 INFO [M:0;9840c0493aab:41337 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T22:44:44,865 INFO [M:0;9840c0493aab:41337 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41337 2024-12-04T22:44:44,866 INFO [M:0;9840c0493aab:41337 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T22:44:44,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:44,974 INFO [M:0;9840c0493aab:41337 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T22:44:44,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41337-0x101a284c1bc0000, quorum=127.0.0.1:52727, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:44,980 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T22:44:45,004 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T22:44:45,004 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:45,016 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1733352284427 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733352284434 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/9840c0493aab:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1733352282571 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/9840c0493aab:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1733352284118 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/9840c0493aab:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/9840c0493aab:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=223 (was 220) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11160 (was 11611) 2024-12-04T22:44:45,024 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=223, ProcessCount=11, AvailableMemoryMB=11159 2024-12-04T22:44:45,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/hadoop-log-dir so I do NOT create it in target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f 2024-12-04T22:44:45,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/hadoop-tmp-dir so I do NOT create it in target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f 2024-12-04T22:44:45,024 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:45,028 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/cluster_66629adf-d3f7-30ff-5c24-c83e147ea34f, deleteOnExit=true 2024-12-04T22:44:45,033 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/cluster_66629adf-d3f7-30ff-5c24-c83e147ea34f/zookeeper_0, clientPort=59666, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/cluster_66629adf-d3f7-30ff-5c24-c83e147ea34f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/cluster_66629adf-d3f7-30ff-5c24-c83e147ea34f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T22:44:45,035 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59666 2024-12-04T22:44:45,053 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b with version=8 2024-12-04T22:44:45,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/hbase-staging 2024-12-04T22:44:45,057 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9840c0493aab:0 server-side Connection retries=45 2024-12-04T22:44:45,057 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:45,058 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:45,058 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T22:44:45,058 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:45,058 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T22:44:45,059 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T22:44:45,059 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T22:44:45,060 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44997 2024-12-04T22:44:45,062 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44997 connecting to ZooKeeper ensemble=127.0.0.1:59666 2024-12-04T22:44:45,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:449970x0, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T22:44:45,115 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44997-0x101a284df120000 connected 2024-12-04T22:44:45,204 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:45,205 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:45,208 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T22:44:45,208 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b, hbase.cluster.distributed=false 2024-12-04T22:44:45,210 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T22:44:45,211 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44997 2024-12-04T22:44:45,211 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44997 2024-12-04T22:44:45,216 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44997 2024-12-04T22:44:45,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44997 2024-12-04T22:44:45,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44997 2024-12-04T22:44:45,234 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9840c0493aab:0 server-side Connection retries=45 2024-12-04T22:44:45,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:45,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:45,234 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T22:44:45,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:45,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T22:44:45,234 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T22:44:45,234 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T22:44:45,235 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33479 2024-12-04T22:44:45,237 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33479 connecting to ZooKeeper ensemble=127.0.0.1:59666 2024-12-04T22:44:45,237 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:45,238 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:45,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334790x0, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T22:44:45,257 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:334790x0, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T22:44:45,257 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33479-0x101a284df120001 connected 2024-12-04T22:44:45,257 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T22:44:45,258 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.cache 2024-12-04T22:44:45,258 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-04T22:44:45,259 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-04T22:44:45,259 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-04T22:44:45,260 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-04T22:44:45,260 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-04T22:44:45,260 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-04T22:44:45,261 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-04T22:44:45,261 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.persistence is complete. 2024-12-04T22:44:45,261 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T22:44:45,262 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T22:44:45,263 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T22:44:45,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33479 2024-12-04T22:44:45,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33479 2024-12-04T22:44:45,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33479 2024-12-04T22:44:45,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33479 2024-12-04T22:44:45,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33479 2024-12-04T22:44:45,297 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9840c0493aab:0 server-side Connection retries=45 2024-12-04T22:44:45,297 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:45,297 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:45,297 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T22:44:45,297 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:45,297 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T22:44:45,298 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T22:44:45,298 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T22:44:45,299 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38659 2024-12-04T22:44:45,301 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38659 connecting to ZooKeeper ensemble=127.0.0.1:59666 2024-12-04T22:44:45,301 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:45,301 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:45,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386590x0, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T22:44:45,316 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:386590x0, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T22:44:45,316 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T22:44:45,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38659-0x101a284df120002 connected 2024-12-04T22:44:45,332 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.cache 2024-12-04T22:44:45,333 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-04T22:44:45,333 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-04T22:44:45,333 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-04T22:44:45,340 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-04T22:44:45,341 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-04T22:44:45,341 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-04T22:44:45,341 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.persistence is complete. 2024-12-04T22:44:45,344 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-04T22:44:45,347 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T22:44:45,348 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T22:44:45,350 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T22:44:45,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38659 2024-12-04T22:44:45,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38659 2024-12-04T22:44:45,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38659 2024-12-04T22:44:45,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38659 2024-12-04T22:44:45,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38659 2024-12-04T22:44:45,396 DEBUG [M:0;9840c0493aab:44997 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9840c0493aab:44997 2024-12-04T22:44:45,397 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9840c0493aab,44997,1733352285056 2024-12-04T22:44:45,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T22:44:45,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T22:44:45,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T22:44:45,412 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9840c0493aab,44997,1733352285056 2024-12-04T22:44:45,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T22:44:45,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T22:44:45,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,426 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T22:44:45,427 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9840c0493aab,44997,1733352285056 from backup master directory 2024-12-04T22:44:45,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T22:44:45,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9840c0493aab,44997,1733352285056 2024-12-04T22:44:45,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T22:44:45,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T22:44:45,436 WARN [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T22:44:45,436 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9840c0493aab,44997,1733352285056 2024-12-04T22:44:45,437 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/hbase.id] with ID: b2f6f687-6db1-4545-87e2-0aa38d98bd22 2024-12-04T22:44:45,438 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/.tmp/hbase.id 2024-12-04T22:44:45,447 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/hbase.id] 2024-12-04T22:44:45,493 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:45,493 INFO [master/9840c0493aab:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T22:44:45,494 INFO [master/9840c0493aab:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-04T22:44:45,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,513 INFO [master/9840c0493aab:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T22:44:45,515 INFO [master/9840c0493aab:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T22:44:45,515 INFO [master/9840c0493aab:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T22:44:45,540 INFO [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store 2024-12-04T22:44:45,547 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:45,547 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T22:44:45,547 INFO [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:45,547 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:45,547 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T22:44:45,547 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:45,548 INFO [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:45,548 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733352285547Disabling compacts and flushes for region at 1733352285547Disabling writes for close at 1733352285547Writing region close event to WAL at 1733352285548 (+1 ms)Closed at 1733352285548 2024-12-04T22:44:45,548 WARN [master/9840c0493aab:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/.initializing 2024-12-04T22:44:45,548 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/WALs/9840c0493aab,44997,1733352285056 2024-12-04T22:44:45,555 INFO [master/9840c0493aab:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T22:44:45,560 INFO [master/9840c0493aab:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9840c0493aab%2C44997%2C1733352285056, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/WALs/9840c0493aab,44997,1733352285056, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/oldWALs, maxLogs=10 2024-12-04T22:44:45,589 INFO [master/9840c0493aab:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/WALs/9840c0493aab,44997,1733352285056/9840c0493aab%2C44997%2C1733352285056.1733352285560 2024-12-04T22:44:45,589 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T22:44:45,589 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T22:44:45,590 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:45,590 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:45,590 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:45,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:45,602 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T22:44:45,603 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:45,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:45,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:45,613 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T22:44:45,613 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:45,613 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T22:44:45,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:45,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T22:44:45,620 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:45,620 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T22:44:45,620 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:45,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T22:44:45,629 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:45,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T22:44:45,630 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:45,630 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:45,631 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:45,631 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:45,631 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:45,631 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T22:44:45,632 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T22:44:45,640 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T22:44:45,640 INFO [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74010356, jitterRate=0.10284024477005005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T22:44:45,641 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733352285590Initializing all the Stores at 1733352285591 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352285591Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352285591Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352285591Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352285591Cleaning up temporary data from old regions at 1733352285631 (+40 ms)Region opened successfully at 1733352285641 (+10 ms) 2024-12-04T22:44:45,642 INFO [master/9840c0493aab:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T22:44:45,646 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586b96d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9840c0493aab/172.17.0.2:0 2024-12-04T22:44:45,648 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T22:44:45,648 INFO [master/9840c0493aab:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T22:44:45,648 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T22:44:45,648 INFO [master/9840c0493aab:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T22:44:45,649 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T22:44:45,649 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T22:44:45,649 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T22:44:45,653 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T22:44:45,654 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T22:44:45,667 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T22:44:45,668 INFO [master/9840c0493aab:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T22:44:45,670 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T22:44:45,678 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T22:44:45,679 INFO [master/9840c0493aab:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T22:44:45,680 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T22:44:45,688 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T22:44:45,690 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T22:44:45,699 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T22:44:45,702 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T22:44:45,715 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T22:44:45,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T22:44:45,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T22:44:45,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T22:44:45,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,727 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9840c0493aab,44997,1733352285056, sessionid=0x101a284df120000, setting cluster-up flag (Was=false) 2024-12-04T22:44:45,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,783 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T22:44:45,785 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9840c0493aab,44997,1733352285056 2024-12-04T22:44:45,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:45,836 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T22:44:45,837 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9840c0493aab,44997,1733352285056 2024-12-04T22:44:45,838 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T22:44:45,840 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T22:44:45,840 INFO [master/9840c0493aab:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T22:44:45,841 INFO [master/9840c0493aab:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T22:44:45,841 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9840c0493aab,44997,1733352285056 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T22:44:45,843 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9840c0493aab:0, corePoolSize=5, maxPoolSize=5 2024-12-04T22:44:45,843 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9840c0493aab:0, corePoolSize=5, maxPoolSize=5 2024-12-04T22:44:45,843 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9840c0493aab:0, corePoolSize=5, maxPoolSize=5 2024-12-04T22:44:45,843 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9840c0493aab:0, corePoolSize=5, maxPoolSize=5 2024-12-04T22:44:45,843 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9840c0493aab:0, corePoolSize=10, maxPoolSize=10 2024-12-04T22:44:45,844 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:45,844 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9840c0493aab:0, corePoolSize=2, maxPoolSize=2 2024-12-04T22:44:45,844 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:45,846 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T22:44:45,846 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T22:44:45,847 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:45,847 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T22:44:45,852 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733352315852 2024-12-04T22:44:45,852 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T22:44:45,853 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T22:44:45,853 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T22:44:45,853 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T22:44:45,853 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T22:44:45,853 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T22:44:45,857 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:45,857 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T22:44:45,857 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T22:44:45,858 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T22:44:45,858 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T22:44:45,858 INFO [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T22:44:45,860 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9840c0493aab:0:becomeActiveMaster-HFileCleaner.large.0-1733352285858,5,FailOnTimeoutGroup] 2024-12-04T22:44:45,861 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9840c0493aab:0:becomeActiveMaster-HFileCleaner.small.0-1733352285860,5,FailOnTimeoutGroup] 2024-12-04T22:44:45,861 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:45,861 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T22:44:45,861 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:45,861 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:45,872 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T22:44:45,872 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b 2024-12-04T22:44:45,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:45,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-04T22:44:45,887 INFO [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(746): ClusterId : b2f6f687-6db1-4545-87e2-0aa38d98bd22 2024-12-04T22:44:45,887 INFO [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(746): ClusterId : b2f6f687-6db1-4545-87e2-0aa38d98bd22 2024-12-04T22:44:45,887 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T22:44:45,887 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T22:44:45,893 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T22:44:45,893 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:45,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:45,894 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-04T22:44:45,900 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T22:44:45,900 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T22:44:45,900 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T22:44:45,900 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T22:44:45,912 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T22:44:45,912 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:45,913 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:45,913 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-04T22:44:45,921 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T22:44:45,921 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T22:44:45,921 DEBUG [RS:0;9840c0493aab:33479 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@698341c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9840c0493aab/172.17.0.2:0 2024-12-04T22:44:45,922 DEBUG [RS:1;9840c0493aab:38659 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4993dc06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9840c0493aab/172.17.0.2:0 2024-12-04T22:44:45,925 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T22:44:45,925 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:45,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:45,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-04T22:44:45,938 DEBUG [RS:0;9840c0493aab:33479 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9840c0493aab:33479 2024-12-04T22:44:45,938 INFO [RS:0;9840c0493aab:33479 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T22:44:45,938 INFO [RS:0;9840c0493aab:33479 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T22:44:45,938 DEBUG [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T22:44:45,939 INFO [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(2659): reportForDuty to master=9840c0493aab,44997,1733352285056 with port=33479, startcode=1733352285233 2024-12-04T22:44:45,940 DEBUG [RS:0;9840c0493aab:33479 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T22:44:45,943 DEBUG [RS:1;9840c0493aab:38659 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9840c0493aab:38659 2024-12-04T22:44:45,944 INFO [RS:1;9840c0493aab:38659 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T22:44:45,944 INFO [RS:1;9840c0493aab:38659 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T22:44:45,944 DEBUG [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T22:44:45,945 INFO [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(2659): reportForDuty to master=9840c0493aab,44997,1733352285056 with port=38659, startcode=1733352285296 2024-12-04T22:44:45,945 DEBUG [RS:1;9840c0493aab:38659 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T22:44:45,949 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T22:44:45,949 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:45,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:45,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T22:44:45,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740 2024-12-04T22:44:45,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740 2024-12-04T22:44:45,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T22:44:45,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T22:44:45,952 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T22:44:45,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T22:44:45,958 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33673, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T22:44:45,959 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41797, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T22:44:45,960 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44997 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9840c0493aab,38659,1733352285296 2024-12-04T22:44:45,960 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44997 {}] master.ServerManager(517): Registering regionserver=9840c0493aab,38659,1733352285296 2024-12-04T22:44:45,962 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44997 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9840c0493aab,33479,1733352285233 2024-12-04T22:44:45,962 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44997 {}] master.ServerManager(517): Registering regionserver=9840c0493aab,33479,1733352285233 2024-12-04T22:44:45,964 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T22:44:45,965 DEBUG [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b 2024-12-04T22:44:45,965 DEBUG [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-04T22:44:45,965 DEBUG [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T22:44:45,965 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63252419, jitterRate=-0.057465508580207825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T22:44:45,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733352285878Initializing all the Stores at 1733352285879 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352285879Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352285879Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352285879Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352285879Cleaning up temporary data from old regions at 1733352285951 (+72 ms)Region opened successfully at 1733352285966 (+15 ms) 2024-12-04T22:44:45,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T22:44:45,966 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T22:44:45,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T22:44:45,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T22:44:45,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T22:44:45,967 DEBUG [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b 2024-12-04T22:44:45,967 DEBUG [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-04T22:44:45,967 DEBUG [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T22:44:45,968 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T22:44:45,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733352285966Disabling compacts and flushes for region at 1733352285966Disabling writes for close at 1733352285966Writing region close event to WAL at 1733352285968 (+2 ms)Closed at 1733352285968 2024-12-04T22:44:45,970 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T22:44:45,970 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T22:44:45,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T22:44:45,973 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T22:44:45,975 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T22:44:45,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T22:44:46,022 DEBUG [RS:0;9840c0493aab:33479 {}] zookeeper.ZKUtil(111): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9840c0493aab,33479,1733352285233 2024-12-04T22:44:46,022 WARN [RS:0;9840c0493aab:33479 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T22:44:46,022 INFO [RS:0;9840c0493aab:33479 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T22:44:46,022 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9840c0493aab,33479,1733352285233] 2024-12-04T22:44:46,023 DEBUG [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,33479,1733352285233 2024-12-04T22:44:46,023 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9840c0493aab,38659,1733352285296] 2024-12-04T22:44:46,023 DEBUG [RS:1;9840c0493aab:38659 {}] zookeeper.ZKUtil(111): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9840c0493aab,38659,1733352285296 2024-12-04T22:44:46,023 WARN [RS:1;9840c0493aab:38659 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T22:44:46,023 INFO [RS:1;9840c0493aab:38659 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T22:44:46,023 DEBUG [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,38659,1733352285296 2024-12-04T22:44:46,030 INFO [RS:0;9840c0493aab:33479 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T22:44:46,034 INFO [RS:0;9840c0493aab:33479 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T22:44:46,034 INFO [RS:0;9840c0493aab:33479 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T22:44:46,034 INFO [RS:0;9840c0493aab:33479 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,036 INFO [RS:1;9840c0493aab:38659 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T22:44:46,036 INFO [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T22:44:46,037 INFO [RS:0;9840c0493aab:33479 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T22:44:46,037 INFO [RS:0;9840c0493aab:33479 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,038 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,038 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,039 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,039 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,039 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,039 INFO [RS:1;9840c0493aab:38659 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T22:44:46,039 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9840c0493aab:0, corePoolSize=2, maxPoolSize=2 2024-12-04T22:44:46,040 INFO [RS:1;9840c0493aab:38659 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T22:44:46,040 INFO [RS:1;9840c0493aab:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,040 INFO [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T22:44:46,041 INFO [RS:1;9840c0493aab:38659 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T22:44:46,041 INFO [RS:1;9840c0493aab:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,041 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,041 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,041 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,041 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,041 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,041 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,042 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,042 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,042 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,042 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,042 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9840c0493aab:0, corePoolSize=2, maxPoolSize=2 2024-12-04T22:44:46,042 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,042 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,042 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,042 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9840c0493aab:0, corePoolSize=3, maxPoolSize=3 2024-12-04T22:44:46,042 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,042 DEBUG [RS:0;9840c0493aab:33479 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9840c0493aab:0, corePoolSize=3, maxPoolSize=3 2024-12-04T22:44:46,042 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,042 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,042 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:46,042 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9840c0493aab:0, corePoolSize=3, maxPoolSize=3 2024-12-04T22:44:46,042 DEBUG [RS:1;9840c0493aab:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9840c0493aab:0, corePoolSize=3, maxPoolSize=3 2024-12-04T22:44:46,060 INFO [RS:0;9840c0493aab:33479 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,061 INFO [RS:0;9840c0493aab:33479 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,061 INFO [RS:0;9840c0493aab:33479 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,061 INFO [RS:0;9840c0493aab:33479 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,062 INFO [RS:0;9840c0493aab:33479 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,062 INFO [RS:0;9840c0493aab:33479 {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,33479,1733352285233-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T22:44:46,068 INFO [RS:1;9840c0493aab:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,068 INFO [RS:1;9840c0493aab:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,068 INFO [RS:1;9840c0493aab:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,068 INFO [RS:1;9840c0493aab:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,069 INFO [RS:1;9840c0493aab:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,069 INFO [RS:1;9840c0493aab:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,38659,1733352285296-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T22:44:46,091 INFO [RS:0;9840c0493aab:33479 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T22:44:46,091 INFO [RS:0;9840c0493aab:33479 {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,33479,1733352285233-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,091 INFO [RS:0;9840c0493aab:33479 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,092 INFO [RS:0;9840c0493aab:33479 {}] regionserver.Replication(171): 9840c0493aab,33479,1733352285233 started 2024-12-04T22:44:46,094 INFO [RS:1;9840c0493aab:38659 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T22:44:46,094 INFO [RS:1;9840c0493aab:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,38659,1733352285296-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,095 INFO [RS:1;9840c0493aab:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,095 INFO [RS:1;9840c0493aab:38659 {}] regionserver.Replication(171): 9840c0493aab,38659,1733352285296 started 2024-12-04T22:44:46,107 INFO [RS:0;9840c0493aab:33479 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,108 INFO [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(1482): Serving as 9840c0493aab,33479,1733352285233, RpcServer on 9840c0493aab/172.17.0.2:33479, sessionid=0x101a284df120001 2024-12-04T22:44:46,108 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T22:44:46,108 DEBUG [RS:0;9840c0493aab:33479 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9840c0493aab,33479,1733352285233 2024-12-04T22:44:46,108 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9840c0493aab,33479,1733352285233' 2024-12-04T22:44:46,108 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T22:44:46,109 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T22:44:46,110 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T22:44:46,110 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T22:44:46,110 DEBUG [RS:0;9840c0493aab:33479 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9840c0493aab,33479,1733352285233 2024-12-04T22:44:46,110 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9840c0493aab,33479,1733352285233' 2024-12-04T22:44:46,110 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T22:44:46,111 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T22:44:46,112 DEBUG [RS:0;9840c0493aab:33479 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T22:44:46,112 INFO [RS:0;9840c0493aab:33479 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T22:44:46,112 INFO [RS:0;9840c0493aab:33479 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T22:44:46,112 INFO [RS:1;9840c0493aab:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,113 INFO [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(1482): Serving as 9840c0493aab,38659,1733352285296, RpcServer on 9840c0493aab/172.17.0.2:38659, sessionid=0x101a284df120002 2024-12-04T22:44:46,113 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T22:44:46,113 DEBUG [RS:1;9840c0493aab:38659 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9840c0493aab,38659,1733352285296 2024-12-04T22:44:46,113 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9840c0493aab,38659,1733352285296' 2024-12-04T22:44:46,113 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T22:44:46,114 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T22:44:46,114 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T22:44:46,114 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T22:44:46,114 DEBUG [RS:1;9840c0493aab:38659 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9840c0493aab,38659,1733352285296 2024-12-04T22:44:46,114 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9840c0493aab,38659,1733352285296' 2024-12-04T22:44:46,114 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T22:44:46,115 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T22:44:46,115 DEBUG [RS:1;9840c0493aab:38659 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T22:44:46,115 INFO [RS:1;9840c0493aab:38659 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T22:44:46,115 INFO [RS:1;9840c0493aab:38659 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T22:44:46,125 WARN [9840c0493aab:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T22:44:46,213 INFO [RS:0;9840c0493aab:33479 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T22:44:46,213 INFO [RS:0;9840c0493aab:33479 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9840c0493aab%2C33479%2C1733352285233, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,33479,1733352285233, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs, maxLogs=55 2024-12-04T22:44:46,216 INFO [RS:1;9840c0493aab:38659 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T22:44:46,217 INFO [RS:1;9840c0493aab:38659 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9840c0493aab%2C38659%2C1733352285296, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,38659,1733352285296, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs, maxLogs=55 2024-12-04T22:44:46,236 INFO [RS:0;9840c0493aab:33479 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,33479,1733352285233/9840c0493aab%2C33479%2C1733352285233.1733352286215 2024-12-04T22:44:46,236 INFO [RS:1;9840c0493aab:38659 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,38659,1733352285296/9840c0493aab%2C38659%2C1733352285296.1733352286218 2024-12-04T22:44:46,236 DEBUG [RS:0;9840c0493aab:33479 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T22:44:46,236 DEBUG [RS:1;9840c0493aab:38659 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T22:44:46,375 DEBUG [9840c0493aab:44997 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-04T22:44:46,376 DEBUG [9840c0493aab:44997 {}] balancer.BalancerClusterState(204): Hosts are {9840c0493aab=0} racks are {/default-rack=0} 2024-12-04T22:44:46,376 DEBUG [9840c0493aab:44997 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T22:44:46,377 DEBUG [9840c0493aab:44997 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T22:44:46,377 DEBUG [9840c0493aab:44997 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T22:44:46,377 DEBUG [9840c0493aab:44997 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T22:44:46,377 INFO [9840c0493aab:44997 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T22:44:46,377 INFO [9840c0493aab:44997 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T22:44:46,377 DEBUG [9840c0493aab:44997 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T22:44:46,377 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9840c0493aab,38659,1733352285296 2024-12-04T22:44:46,378 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9840c0493aab,38659,1733352285296, state=OPENING 2024-12-04T22:44:46,462 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T22:44:46,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:46,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:46,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:46,481 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:46,481 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:46,481 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T22:44:46,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9840c0493aab,38659,1733352285296}] 2024-12-04T22:44:46,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:46,634 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T22:44:46,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55103, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T22:44:46,640 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T22:44:46,640 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T22:44:46,641 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T22:44:46,642 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9840c0493aab%2C38659%2C1733352285296.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,38659,1733352285296, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs, maxLogs=55 2024-12-04T22:44:46,666 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,38659,1733352285296/9840c0493aab%2C38659%2C1733352285296.meta.1733352286643.meta 2024-12-04T22:44:46,666 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T22:44:46,666 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T22:44:46,667 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T22:44:46,667 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T22:44:46,667 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T22:44:46,667 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T22:44:46,667 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:46,668 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T22:44:46,668 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T22:44:46,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-04T22:44:46,669 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T22:44:46,670 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:46,670 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:46,670 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-04T22:44:46,670 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T22:44:46,671 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:46,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:46,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-04T22:44:46,671 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T22:44:46,672 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:46,672 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:46,672 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-04T22:44:46,673 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T22:44:46,673 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:46,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:46,673 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T22:44:46,674 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740 2024-12-04T22:44:46,674 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740 2024-12-04T22:44:46,674 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T22:44:46,674 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T22:44:46,675 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T22:44:46,675 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T22:44:46,676 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68083902, jitterRate=0.014529198408126831}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T22:44:46,676 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T22:44:46,677 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733352286668Writing region info on filesystem at 1733352286668Initializing all the Stores at 1733352286668Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352286668Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352286668Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352286669 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352286669Cleaning up temporary data from old regions at 1733352286674 (+5 ms)Running coprocessor post-open hooks at 1733352286676 (+2 ms)Region opened successfully at 1733352286676 2024-12-04T22:44:46,677 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733352286634 2024-12-04T22:44:46,680 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T22:44:46,680 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T22:44:46,680 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9840c0493aab,38659,1733352285296 2024-12-04T22:44:46,681 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9840c0493aab,38659,1733352285296, state=OPEN 2024-12-04T22:44:46,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T22:44:46,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T22:44:46,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T22:44:46,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:46,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:46,694 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9840c0493aab,38659,1733352285296 2024-12-04T22:44:46,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:46,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T22:44:46,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9840c0493aab,38659,1733352285296 in 213 msec 2024-12-04T22:44:46,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T22:44:46,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 726 msec 2024-12-04T22:44:46,700 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T22:44:46,700 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T22:44:46,707 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T22:44:46,707 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9840c0493aab,38659,1733352285296, seqNum=-1] 2024-12-04T22:44:46,707 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T22:44:46,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56039, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T22:44:46,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 875 msec 2024-12-04T22:44:46,716 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733352286716, completionTime=-1 2024-12-04T22:44:46,716 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-04T22:44:46,716 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T22:44:46,719 INFO [master/9840c0493aab:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-04T22:44:46,719 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733352346719 2024-12-04T22:44:46,719 INFO [master/9840c0493aab:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733352406719 2024-12-04T22:44:46,719 INFO [master/9840c0493aab:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-12-04T22:44:46,720 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,44997,1733352285056-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,720 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,44997,1733352285056-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,720 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,44997,1733352285056-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,720 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9840c0493aab:44997, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,720 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,721 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,725 DEBUG [master/9840c0493aab:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T22:44:46,727 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.291sec 2024-12-04T22:44:46,728 INFO [master/9840c0493aab:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T22:44:46,728 INFO [master/9840c0493aab:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T22:44:46,728 INFO [master/9840c0493aab:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T22:44:46,728 INFO [master/9840c0493aab:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T22:44:46,728 INFO [master/9840c0493aab:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T22:44:46,728 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,44997,1733352285056-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T22:44:46,728 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,44997,1733352285056-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T22:44:46,729 DEBUG [master/9840c0493aab:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T22:44:46,729 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T22:44:46,729 INFO [master/9840c0493aab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,44997,1733352285056-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:46,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d25b74f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T22:44:46,787 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9840c0493aab,44997,-1 for getting cluster id 2024-12-04T22:44:46,787 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T22:44:46,790 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b2f6f687-6db1-4545-87e2-0aa38d98bd22' 2024-12-04T22:44:46,790 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T22:44:46,791 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b2f6f687-6db1-4545-87e2-0aa38d98bd22" 2024-12-04T22:44:46,791 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d66f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T22:44:46,792 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9840c0493aab,44997,-1] 2024-12-04T22:44:46,792 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T22:44:46,793 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:46,795 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T22:44:46,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5858d96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T22:44:46,797 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T22:44:46,799 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9840c0493aab,38659,1733352285296, seqNum=-1] 2024-12-04T22:44:46,800 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T22:44:46,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T22:44:46,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9840c0493aab,44997,1733352285056 2024-12-04T22:44:46,805 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T22:44:46,806 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 9840c0493aab,44997,1733352285056 2024-12-04T22:44:46,806 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f44d5d3 2024-12-04T22:44:46,806 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T22:44:46,808 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T22:44:46,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733352286804', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T22:44:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733352286804 2024-12-04T22:44:46,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733352286804 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T22:44:46,822 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:46,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733352286804" procId is: 4 2024-12-04T22:44:46,823 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733352286804 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T22:44:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T22:44:46,835 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T22:44:46,838 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733352286804-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2d5f9af5ebb9b68bbf2f5ba472b0910b, NAME => 'testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733352286804', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b 2024-12-04T22:44:46,870 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733352286804-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:46,870 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733352286804-pool-0 {}] regionserver.HRegion(1722): Closing 2d5f9af5ebb9b68bbf2f5ba472b0910b, disabling compactions & flushes 2024-12-04T22:44:46,870 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733352286804-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:46,870 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733352286804-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:46,870 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733352286804-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. after waiting 0 ms 2024-12-04T22:44:46,870 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733352286804-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:46,871 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733352286804-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:46,871 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733352286804-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2d5f9af5ebb9b68bbf2f5ba472b0910b: Waiting for close lock at 1733352286870Disabling compacts and flushes for region at 1733352286870Disabling writes for close at 1733352286870Writing region close event to WAL at 1733352286871 (+1 ms)Closed at 1733352286871 2024-12-04T22:44:46,879 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733352286804 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T22:44:46,880 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733352286879"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733352286879"}]},"ts":"1733352286879"} 2024-12-04T22:44:46,884 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T22:44:46,885 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733352286804 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T22:44:46,886 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733352286804","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733352286885"}]},"ts":"1733352286885"} 2024-12-04T22:44:46,889 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733352286804, state=ENABLING in hbase:meta 2024-12-04T22:44:46,890 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {9840c0493aab=0} racks are {/default-rack=0} 2024-12-04T22:44:46,891 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T22:44:46,891 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T22:44:46,891 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T22:44:46,891 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T22:44:46,891 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T22:44:46,891 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T22:44:46,891 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T22:44:46,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733352286804, region=2d5f9af5ebb9b68bbf2f5ba472b0910b, ASSIGN}] 2024-12-04T22:44:46,893 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733352286804, region=2d5f9af5ebb9b68bbf2f5ba472b0910b, ASSIGN 2024-12-04T22:44:46,894 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733352286804, region=2d5f9af5ebb9b68bbf2f5ba472b0910b, ASSIGN; state=OFFLINE, location=9840c0493aab,38659,1733352285296; forceNewPlan=false, retain=false 2024-12-04T22:44:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T22:44:47,045 INFO [9840c0493aab:44997 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T22:44:47,046 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2d5f9af5ebb9b68bbf2f5ba472b0910b, regionState=OPENING, regionLocation=9840c0493aab,38659,1733352285296 2024-12-04T22:44:47,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733352286804, region=2d5f9af5ebb9b68bbf2f5ba472b0910b, ASSIGN because future has completed 2024-12-04T22:44:47,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d5f9af5ebb9b68bbf2f5ba472b0910b, server=9840c0493aab,38659,1733352285296}] 2024-12-04T22:44:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T22:44:47,207 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:47,207 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d5f9af5ebb9b68bbf2f5ba472b0910b, NAME => 'testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b.', STARTKEY => '', ENDKEY => ''} 2024-12-04T22:44:47,207 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733352286804 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:47,207 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:47,208 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:47,208 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:47,209 INFO [StoreOpener-2d5f9af5ebb9b68bbf2f5ba472b0910b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:47,212 INFO [StoreOpener-2d5f9af5ebb9b68bbf2f5ba472b0910b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d5f9af5ebb9b68bbf2f5ba472b0910b columnFamilyName family 2024-12-04T22:44:47,213 DEBUG [StoreOpener-2d5f9af5ebb9b68bbf2f5ba472b0910b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:47,213 INFO [StoreOpener-2d5f9af5ebb9b68bbf2f5ba472b0910b-1 {}] regionserver.HStore(327): Store=2d5f9af5ebb9b68bbf2f5ba472b0910b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T22:44:47,213 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:47,214 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:47,214 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:47,215 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:47,215 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:47,216 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:47,222 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T22:44:47,222 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2d5f9af5ebb9b68bbf2f5ba472b0910b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62367712, jitterRate=-0.0706486701965332}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T22:44:47,222 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:47,223 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2d5f9af5ebb9b68bbf2f5ba472b0910b: Running coprocessor pre-open hook at 1733352287208Writing region info on filesystem at 1733352287208Initializing all the Stores at 1733352287208Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352287208Cleaning up temporary data from old regions at 1733352287215 (+7 ms)Running coprocessor post-open hooks at 1733352287222 (+7 ms)Region opened successfully at 1733352287223 (+1 ms) 2024-12-04T22:44:47,224 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b., pid=6, masterSystemTime=1733352287204 2024-12-04T22:44:47,226 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:47,227 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:47,227 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2d5f9af5ebb9b68bbf2f5ba472b0910b, regionState=OPEN, openSeqNum=2, regionLocation=9840c0493aab,38659,1733352285296 2024-12-04T22:44:47,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d5f9af5ebb9b68bbf2f5ba472b0910b, server=9840c0493aab,38659,1733352285296 because future has completed 2024-12-04T22:44:47,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T22:44:47,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d5f9af5ebb9b68bbf2f5ba472b0910b, server=9840c0493aab,38659,1733352285296 in 181 msec 2024-12-04T22:44:47,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T22:44:47,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733352286804, region=2d5f9af5ebb9b68bbf2f5ba472b0910b, ASSIGN in 344 msec 2024-12-04T22:44:47,238 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733352286804 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T22:44:47,238 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733352286804","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733352287238"}]},"ts":"1733352287238"} 2024-12-04T22:44:47,241 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733352286804, state=ENABLED in hbase:meta 2024-12-04T22:44:47,242 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733352286804 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T22:44:47,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733352286804 in 433 msec 2024-12-04T22:44:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T22:44:47,451 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733352286804 completed 2024-12-04T22:44:47,451 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733352286804 get assigned. Timeout = 60000ms 2024-12-04T22:44:47,451 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T22:44:47,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733352286804 assigned to meta. Checking AM states. 2024-12-04T22:44:47,455 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T22:44:47,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733352286804 assigned. 2024-12-04T22:44:47,459 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733352286804', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b., hostname=9840c0493aab,38659,1733352285296, seqNum=2] 2024-12-04T22:44:47,464 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2d5f9af5ebb9b68bbf2f5ba472b0910b 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-04T22:44:47,474 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1f7ea4a615fb4134a327ddc266c998aa 2024-12-04T22:44:47,489 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/.tmp/family/1f7ea4a615fb4134a327ddc266c998aa is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-04T22:44:47,489 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/.tmp/family/1f7ea4a615fb4134a327ddc266c998aa 2024-12-04T22:44:47,492 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/.tmp/family/1f7ea4a615fb4134a327ddc266c998aa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa 2024-12-04T22:44:47,493 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa, delay=928 ms 2024-12-04T22:44:47,494 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa, entries=2, sequenceid=6, filesize=4.9 K 2024-12-04T22:44:47,495 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 2d5f9af5ebb9b68bbf2f5ba472b0910b in 31ms, sequenceid=6, compaction requested=false 2024-12-04T22:44:47,495 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733352286804' 2024-12-04T22:44:47,496 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2d5f9af5ebb9b68bbf2f5ba472b0910b: 2024-12-04T22:44:48,220 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T22:44:48,221 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T22:44:48,223 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T22:44:48,223 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T22:44:48,223 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T22:44:48,224 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T22:44:48,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733352286804 2024-12-04T22:44:48,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733352286804 Metrics about Tables on a single HBase RegionServer 2024-12-04T22:44:48,422 DEBUG [hfile-prefetch-1733352284118 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1f7ea4a615fb4134a327ddc266c998aa_0. 2024-12-04T22:44:48,423 DEBUG [hfile-prefetch-1733352284118 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1f7ea4a615fb4134a327ddc266c998aa_117. 2024-12-04T22:44:48,423 DEBUG [hfile-prefetch-1733352284118 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa, with total blocks 2, and data blocks 1 2024-12-04T22:44:48,424 DEBUG [hfile-prefetch-1733352284118 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-04T22:44:48,424 DEBUG [hfile-prefetch-1733352284118 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-04T22:44:48,424 DEBUG [hfile-prefetch-1733352284118 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa. Will sleep 100ms and try the verification again. 2024-12-04T22:44:48,496 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;9840c0493aab:38659,5,FailOnTimeoutGroup] 2024-12-04T22:44:48,496 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9840c0493aab,38659,1733352285296' ***** 2024-12-04T22:44:48,497 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-04T22:44:48,497 INFO [RS:1;9840c0493aab:38659 {}] bucket.BucketCache(1738): Disabling cache 2024-12-04T22:44:48,524 DEBUG [hfile-prefetch-1733352284118 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa, with total blocks 2, and data blocks 1 2024-12-04T22:44:48,524 DEBUG [hfile-prefetch-1733352284118 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-04T22:44:48,524 DEBUG [hfile-prefetch-1733352284118 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-04T22:44:48,524 DEBUG [hfile-prefetch-1733352284118 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa. Will sleep 100ms and try the verification again. 2024-12-04T22:44:48,562 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T22:44:48,563 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-04T22:44:48,563 INFO [RS:1;9840c0493aab:38659 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.persistence 2024-12-04T22:44:48,563 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-04T22:44:48,564 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-04T22:44:48,564 INFO [RS:1;9840c0493aab:38659 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-04T22:44:48,564 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-04T22:44:48,574 DEBUG [RS:1;9840c0493aab:38659 {}] bucket.BucketCache(1487): Thread RS:1;9840c0493aab:38659 started persisting bucket cache to file 2024-12-04T22:44:48,574 DEBUG [RS:1;9840c0493aab:38659 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T22:44:48,574 DEBUG [RS:1;9840c0493aab:38659 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T22:44:48,578 DEBUG [RS:1;9840c0493aab:38659 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-04T22:44:48,578 DEBUG [RS:1;9840c0493aab:38659 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733352288574 2024-12-04T22:44:48,578 DEBUG [RS:1;9840c0493aab:38659 {}] bucket.BucketCache(1509): Thread RS:1;9840c0493aab:38659 finished persisting bucket cache to file, renaming 2024-12-04T22:44:48,579 INFO [RS:1;9840c0493aab:38659 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T22:44:48,579 INFO [RS:1;9840c0493aab:38659 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T22:44:48,579 INFO [RS:1;9840c0493aab:38659 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T22:44:48,579 INFO [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:48,579 INFO [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(959): stopping server 9840c0493aab,38659,1733352285296 2024-12-04T22:44:48,579 INFO [RS:1;9840c0493aab:38659 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T22:44:48,579 INFO [RS:1;9840c0493aab:38659 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9840c0493aab:38659. 2024-12-04T22:44:48,579 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-04T22:44:48,579 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2d5f9af5ebb9b68bbf2f5ba472b0910b, disabling compactions & flushes 2024-12-04T22:44:48,580 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:48,580 DEBUG [RS:1;9840c0493aab:38659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T22:44:48,580 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:48,580 DEBUG [RS:1;9840c0493aab:38659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:48,580 INFO [regionserver/9840c0493aab:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T22:44:48,580 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. after waiting 0 ms 2024-12-04T22:44:48,580 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:48,580 INFO [RS:1;9840c0493aab:38659 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T22:44:48,580 INFO [RS:1;9840c0493aab:38659 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T22:44:48,580 INFO [RS:1;9840c0493aab:38659 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T22:44:48,580 INFO [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T22:44:48,581 INFO [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T22:44:48,581 DEBUG [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(1325): Online Regions={2d5f9af5ebb9b68bbf2f5ba472b0910b=testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T22:44:48,581 DEBUG [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:48,581 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-04T22:44:48,581 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T22:44:48,581 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T22:44:48,581 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T22:44:48,581 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T22:44:48,581 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T22:44:48,582 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-12-04T22:44:48,584 DEBUG [StoreFileCloser-2d5f9af5ebb9b68bbf2f5ba472b0910b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa 2024-12-04T22:44:48,584 WARN [hfile-prefetch-1733352284118 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 10 more 2024-12-04T22:44:48,585 DEBUG [hfile-prefetch-1733352284118 {}] hfile.PrefetchExecutor(121): Prefetch completed for 1f7ea4a615fb4134a327ddc266c998aa 2024-12-04T22:44:48,592 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-04T22:44:48,592 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:48,593 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2d5f9af5ebb9b68bbf2f5ba472b0910b: Waiting for close lock at 1733352288579Running coprocessor pre-close hooks at 1733352288579Disabling compacts and flushes for region at 1733352288579Disabling writes for close at 1733352288580 (+1 ms)Writing region close event to WAL at 1733352288585 (+5 ms)Running coprocessor post-close hooks at 1733352288592 (+7 ms)Closed at 1733352288592 2024-12-04T22:44:48,593 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:48,594 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 411021807b994048bdb000067667e1cd 2024-12-04T22:44:48,615 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/info/411021807b994048bdb000067667e1cd is 209, key is testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b./info:regioninfo/1733352287227/Put/seqid=0 2024-12-04T22:44:48,616 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/info/411021807b994048bdb000067667e1cd 2024-12-04T22:44:48,629 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f85bf213b7494a738d81ae070bd78aa5 2024-12-04T22:44:48,644 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/ns/f85bf213b7494a738d81ae070bd78aa5 is 43, key is default/ns:d/1733352286710/Put/seqid=0 2024-12-04T22:44:48,644 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/ns/f85bf213b7494a738d81ae070bd78aa5 2024-12-04T22:44:48,656 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5029ae9eca094cd9b75fdf92528d8c19 2024-12-04T22:44:48,661 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-04T22:44:48,661 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-04T22:44:48,662 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T22:44:48,662 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T22:44:48,665 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T22:44:48,665 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733352288661 2024-12-04T22:44:48,665 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-04T22:44:48,672 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/table/5029ae9eca094cd9b75fdf92528d8c19 is 80, key is testBlockEvictionOnGracefulStop1733352286804/table:state/1733352287238/Put/seqid=0 2024-12-04T22:44:48,672 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/table/5029ae9eca094cd9b75fdf92528d8c19 2024-12-04T22:44:48,676 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/info/411021807b994048bdb000067667e1cd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/411021807b994048bdb000067667e1cd 2024-12-04T22:44:48,677 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/411021807b994048bdb000067667e1cd, delay=942 ms 2024-12-04T22:44:48,679 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/411021807b994048bdb000067667e1cd, entries=10, sequenceid=11, filesize=7.0 K 2024-12-04T22:44:48,679 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/ns/f85bf213b7494a738d81ae070bd78aa5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/ns/f85bf213b7494a738d81ae070bd78aa5 2024-12-04T22:44:48,681 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/ns/f85bf213b7494a738d81ae070bd78aa5, delay=974 ms 2024-12-04T22:44:48,682 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/ns/f85bf213b7494a738d81ae070bd78aa5, entries=2, sequenceid=11, filesize=5.0 K 2024-12-04T22:44:48,682 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/table/5029ae9eca094cd9b75fdf92528d8c19 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/table/5029ae9eca094cd9b75fdf92528d8c19 2024-12-04T22:44:48,684 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/table/5029ae9eca094cd9b75fdf92528d8c19, delay=979 ms 2024-12-04T22:44:48,685 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/table/5029ae9eca094cd9b75fdf92528d8c19, entries=2, sequenceid=11, filesize=5.3 K 2024-12-04T22:44:48,686 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=11, compaction requested=false 2024-12-04T22:44:48,686 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/411021807b994048bdb000067667e1cd 2024-12-04T22:44:48,700 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/ns/f85bf213b7494a738d81ae070bd78aa5 2024-12-04T22:44:48,701 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/table/5029ae9eca094cd9b75fdf92528d8c19 2024-12-04T22:44:48,706 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-04T22:44:48,706 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T22:44:48,706 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T22:44:48,707 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733352288581Running coprocessor pre-close hooks at 1733352288581Disabling compacts and flushes for region at 1733352288581Disabling writes for close at 1733352288581Obtaining lock to block concurrent updates at 1733352288582 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733352288582Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733352288582Flushing stores of hbase:meta,,1.1588230740 at 1733352288583 (+1 ms)Flushing 1588230740/info: creating writer at 1733352288583Flushing 1588230740/info: appending metadata at 1733352288615 (+32 ms)Flushing 1588230740/info: closing flushed file at 1733352288615Flushing 1588230740/ns: creating writer at 1733352288619 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733352288643 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1733352288643Flushing 1588230740/table: creating writer at 1733352288647 (+4 ms)Flushing 1588230740/table: appending metadata at 1733352288671 (+24 ms)Flushing 1588230740/table: closing flushed file at 1733352288671Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66f56be0: reopening flushed file at 1733352288675 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@413021cf: reopening flushed file at 1733352288679 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f67a1c4: reopening flushed file at 1733352288682 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=11, compaction requested=false at 1733352288686 (+4 ms)Writing region close event to WAL at 1733352288702 (+16 ms)Running coprocessor post-close hooks at 1733352288706 (+4 ms)Closed at 1733352288706 2024-12-04T22:44:48,707 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T22:44:48,781 INFO [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(976): stopping server 9840c0493aab,38659,1733352285296; all regions closed. 2024-12-04T22:44:48,783 DEBUG [RS:1;9840c0493aab:38659 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs 2024-12-04T22:44:48,784 INFO [RS:1;9840c0493aab:38659 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9840c0493aab%2C38659%2C1733352285296.meta:.meta(num 1733352286643) 2024-12-04T22:44:48,786 DEBUG [RS:1;9840c0493aab:38659 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs 2024-12-04T22:44:48,786 INFO [RS:1;9840c0493aab:38659 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9840c0493aab%2C38659%2C1733352285296:(num 1733352286218) 2024-12-04T22:44:48,786 DEBUG [RS:1;9840c0493aab:38659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:48,786 INFO [RS:1;9840c0493aab:38659 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T22:44:48,786 INFO [RS:1;9840c0493aab:38659 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T22:44:48,786 INFO [RS:1;9840c0493aab:38659 {}] hbase.ChoreService(370): Chore service for: regionserver/9840c0493aab:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T22:44:48,786 INFO [RS:1;9840c0493aab:38659 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T22:44:48,786 INFO [regionserver/9840c0493aab:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T22:44:48,786 INFO [RS:1;9840c0493aab:38659 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38659 2024-12-04T22:44:48,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9840c0493aab,38659,1733352285296 2024-12-04T22:44:48,804 INFO [RS:1;9840c0493aab:38659 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T22:44:48,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T22:44:48,815 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9840c0493aab,38659,1733352285296] 2024-12-04T22:44:48,825 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9840c0493aab,38659,1733352285296 already deleted, retry=false 2024-12-04T22:44:48,825 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9840c0493aab,38659,1733352285296 on 9840c0493aab,44997,1733352285056 2024-12-04T22:44:48,831 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9840c0493aab,38659,1733352285296, splitWal=true, meta=true 2024-12-04T22:44:48,833 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9840c0493aab,38659,1733352285296, splitWal=true, meta=true 2024-12-04T22:44:48,833 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 9840c0493aab,38659,1733352285296 (carryingMeta=true) 9840c0493aab,38659,1733352285296/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3e3eae54[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-04T22:44:48,834 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 9840c0493aab,38659,1733352285296, splitWal=true, meta=true, isMeta: true 2024-12-04T22:44:48,835 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,38659,1733352285296-splitting 2024-12-04T22:44:48,835 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,38659,1733352285296-splitting dir is empty, no logs to split. 2024-12-04T22:44:48,835 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9840c0493aab,38659,1733352285296 WAL count=0, meta=true 2024-12-04T22:44:48,840 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,38659,1733352285296-splitting dir is empty, no logs to split. 2024-12-04T22:44:48,840 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9840c0493aab,38659,1733352285296 WAL count=0, meta=true 2024-12-04T22:44:48,840 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 9840c0493aab,38659,1733352285296 WAL splitting is done? wals=0, meta=true 2024-12-04T22:44:48,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T22:44:48,844 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T22:44:48,845 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-04T22:44:48,915 INFO [RS:1;9840c0493aab:38659 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T22:44:48,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:48,915 INFO [RS:1;9840c0493aab:38659 {}] regionserver.HRegionServer(1031): Exiting; stopping=9840c0493aab,38659,1733352285296; zookeeper connection closed. 2024-12-04T22:44:48,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101a284df120002, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:48,915 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2a171771 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2a171771 2024-12-04T22:44:48,995 DEBUG [9840c0493aab:44997 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T22:44:48,996 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9840c0493aab,33479,1733352285233 2024-12-04T22:44:48,997 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9840c0493aab,33479,1733352285233, state=OPENING 2024-12-04T22:44:49,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T22:44:49,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T22:44:49,009 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T22:44:49,010 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:49,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9840c0493aab,33479,1733352285233}] 2024-12-04T22:44:49,010 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:49,015 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9840c0493aab:0 server-side Connection retries=45 2024-12-04T22:44:49,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:49,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:49,016 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T22:44:49,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T22:44:49,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T22:44:49,016 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T22:44:49,016 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T22:44:49,017 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35557 2024-12-04T22:44:49,019 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35557 connecting to ZooKeeper ensemble=127.0.0.1:59666 2024-12-04T22:44:49,019 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:49,019 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:49,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355570x0, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T22:44:49,031 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:355570x0, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-04T22:44:49,031 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-04T22:44:49,032 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35557-0x101a284df120003 connected 2024-12-04T22:44:49,032 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T22:44:49,036 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.cache 2024-12-04T22:44:49,037 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-04T22:44:49,037 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-04T22:44:49,037 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-04T22:44:49,037 INFO [Thread-208 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-04T22:44:49,038 INFO [Thread-208 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-04T22:44:49,039 INFO [Thread-208 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-04T22:44:49,040 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-04T22:44:49,040 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T22:44:49,041 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35557-0x101a284df120003, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T22:44:49,044 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35557-0x101a284df120003, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T22:44:49,056 DEBUG [Thread-208 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-04T22:44:49,057 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-04T22:44:49,057 INFO [Thread-208 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-04T22:44:49,057 INFO [Thread-208 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.persistence is complete. 2024-12-04T22:44:49,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35557 2024-12-04T22:44:49,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35557 2024-12-04T22:44:49,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35557 2024-12-04T22:44:49,068 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35557 2024-12-04T22:44:49,069 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35557 2024-12-04T22:44:49,073 INFO [RS:2;9840c0493aab:35557 {}] regionserver.HRegionServer(746): ClusterId : b2f6f687-6db1-4545-87e2-0aa38d98bd22 2024-12-04T22:44:49,073 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T22:44:49,121 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T22:44:49,121 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T22:44:49,131 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T22:44:49,132 DEBUG [RS:2;9840c0493aab:35557 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d251eac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9840c0493aab/172.17.0.2:0 2024-12-04T22:44:49,145 DEBUG [RS:2;9840c0493aab:35557 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;9840c0493aab:35557 2024-12-04T22:44:49,145 INFO [RS:2;9840c0493aab:35557 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T22:44:49,145 INFO [RS:2;9840c0493aab:35557 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T22:44:49,145 DEBUG [RS:2;9840c0493aab:35557 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T22:44:49,146 INFO [RS:2;9840c0493aab:35557 {}] regionserver.HRegionServer(2659): reportForDuty to master=9840c0493aab,44997,1733352285056 with port=35557, startcode=1733352289015 2024-12-04T22:44:49,146 DEBUG [RS:2;9840c0493aab:35557 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T22:44:49,149 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39981, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T22:44:49,149 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44997 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9840c0493aab,35557,1733352289015 2024-12-04T22:44:49,149 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44997 {}] master.ServerManager(517): Registering regionserver=9840c0493aab,35557,1733352289015 2024-12-04T22:44:49,151 DEBUG [RS:2;9840c0493aab:35557 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b 2024-12-04T22:44:49,151 DEBUG [RS:2;9840c0493aab:35557 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-04T22:44:49,151 DEBUG [RS:2;9840c0493aab:35557 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T22:44:49,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T22:44:49,162 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T22:44:49,162 DEBUG [RS:2;9840c0493aab:35557 {}] zookeeper.ZKUtil(111): regionserver:35557-0x101a284df120003, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9840c0493aab,35557,1733352289015 2024-12-04T22:44:49,162 WARN [RS:2;9840c0493aab:35557 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T22:44:49,163 INFO [RS:2;9840c0493aab:35557 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T22:44:49,163 DEBUG [RS:2;9840c0493aab:35557 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,35557,1733352289015 2024-12-04T22:44:49,163 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9840c0493aab,35557,1733352289015] 2024-12-04T22:44:49,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50425, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T22:44:49,166 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T22:44:49,166 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T22:44:49,167 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T22:44:49,168 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9840c0493aab%2C33479%2C1733352285233.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,33479,1733352285233, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs, maxLogs=55 2024-12-04T22:44:49,169 INFO [RS:2;9840c0493aab:35557 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T22:44:49,172 INFO [RS:2;9840c0493aab:35557 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T22:44:49,173 INFO [RS:2;9840c0493aab:35557 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T22:44:49,173 INFO [RS:2;9840c0493aab:35557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:49,173 INFO [RS:2;9840c0493aab:35557 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T22:44:49,174 INFO [RS:2;9840c0493aab:35557 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T22:44:49,174 INFO [RS:2;9840c0493aab:35557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:49,174 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:49,174 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:49,174 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:49,174 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:49,174 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:49,174 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9840c0493aab:0, corePoolSize=2, maxPoolSize=2 2024-12-04T22:44:49,174 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:49,174 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:49,175 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:49,175 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:49,175 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:49,175 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9840c0493aab:0, corePoolSize=1, maxPoolSize=1 2024-12-04T22:44:49,175 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9840c0493aab:0, corePoolSize=3, maxPoolSize=3 2024-12-04T22:44:49,175 DEBUG [RS:2;9840c0493aab:35557 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9840c0493aab:0, corePoolSize=3, maxPoolSize=3 2024-12-04T22:44:49,175 INFO [RS:2;9840c0493aab:35557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:49,175 INFO [RS:2;9840c0493aab:35557 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:49,175 INFO [RS:2;9840c0493aab:35557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:49,176 INFO [RS:2;9840c0493aab:35557 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:49,176 INFO [RS:2;9840c0493aab:35557 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:49,176 INFO [RS:2;9840c0493aab:35557 {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,35557,1733352289015-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T22:44:49,188 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,33479,1733352285233/9840c0493aab%2C33479%2C1733352285233.meta.1733352289169.meta 2024-12-04T22:44:49,188 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T22:44:49,188 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T22:44:49,189 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T22:44:49,189 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T22:44:49,189 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T22:44:49,189 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T22:44:49,189 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:49,189 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T22:44:49,190 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T22:44:49,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-04T22:44:49,191 INFO [RS:2;9840c0493aab:35557 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T22:44:49,191 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T22:44:49,191 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:49,191 INFO [RS:2;9840c0493aab:35557 {}] hbase.ChoreService(168): Chore ScheduledChore name=9840c0493aab,35557,1733352289015-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:49,191 INFO [RS:2;9840c0493aab:35557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:49,191 INFO [RS:2;9840c0493aab:35557 {}] regionserver.Replication(171): 9840c0493aab,35557,1733352289015 started 2024-12-04T22:44:49,193 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/411021807b994048bdb000067667e1cd, delay=985 ms 2024-12-04T22:44:49,194 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/411021807b994048bdb000067667e1cd 2024-12-04T22:44:49,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:49,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-04T22:44:49,196 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T22:44:49,196 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:49,198 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/ns/f85bf213b7494a738d81ae070bd78aa5, delay=923 ms 2024-12-04T22:44:49,200 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/ns/f85bf213b7494a738d81ae070bd78aa5 2024-12-04T22:44:49,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:49,201 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-04T22:44:49,201 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T22:44:49,202 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:49,202 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:49,202 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-04T22:44:49,203 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T22:44:49,203 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:49,205 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/table/5029ae9eca094cd9b75fdf92528d8c19, delay=980 ms 2024-12-04T22:44:49,206 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/table/5029ae9eca094cd9b75fdf92528d8c19 2024-12-04T22:44:49,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T22:44:49,207 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T22:44:49,208 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740 2024-12-04T22:44:49,208 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740 2024-12-04T22:44:49,209 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T22:44:49,209 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T22:44:49,210 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T22:44:49,210 INFO [RS:2;9840c0493aab:35557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T22:44:49,211 INFO [RS:2;9840c0493aab:35557 {}] regionserver.HRegionServer(1482): Serving as 9840c0493aab,35557,1733352289015, RpcServer on 9840c0493aab/172.17.0.2:35557, sessionid=0x101a284df120003 2024-12-04T22:44:49,211 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T22:44:49,211 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T22:44:49,211 DEBUG [RS:2;9840c0493aab:35557 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9840c0493aab,35557,1733352289015 2024-12-04T22:44:49,211 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9840c0493aab,35557,1733352289015' 2024-12-04T22:44:49,211 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T22:44:49,211 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67780870, jitterRate=0.010013669729232788}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T22:44:49,211 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T22:44:49,212 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T22:44:49,213 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733352289190Writing region info on filesystem at 1733352289190Initializing all the Stores at 1733352289190Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352289190Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352289190Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352289190Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733352289190Cleaning up temporary data from old regions at 1733352289209 (+19 ms)Running coprocessor post-open hooks at 1733352289211 (+2 ms)Region opened successfully at 1733352289212 (+1 ms) 2024-12-04T22:44:49,213 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T22:44:49,213 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T22:44:49,213 DEBUG [RS:2;9840c0493aab:35557 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9840c0493aab,35557,1733352289015 2024-12-04T22:44:49,213 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9840c0493aab,35557,1733352289015' 2024-12-04T22:44:49,213 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T22:44:49,214 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T22:44:49,215 DEBUG [RS:2;9840c0493aab:35557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T22:44:49,215 INFO [RS:2;9840c0493aab:35557 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T22:44:49,215 INFO [RS:2;9840c0493aab:35557 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T22:44:49,216 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733352289162 2024-12-04T22:44:49,220 DEBUG [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T22:44:49,220 INFO [RS_OPEN_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T22:44:49,220 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=9840c0493aab,33479,1733352285233 2024-12-04T22:44:49,221 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9840c0493aab,33479,1733352285233, state=OPEN 2024-12-04T22:44:49,229 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44997 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=9840c0493aab,33479,1733352285233, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-04T22:44:49,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T22:44:49,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T22:44:49,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35557-0x101a284df120003, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T22:44:49,231 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9840c0493aab,33479,1733352285233 2024-12-04T22:44:49,231 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:49,231 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:49,231 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T22:44:49,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-04T22:44:49,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9840c0493aab,33479,1733352285233 in 222 msec 2024-12-04T22:44:49,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T22:44:49,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 392 msec 2024-12-04T22:44:49,239 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 9840c0493aab,38659,1733352285296 had 2 regions 2024-12-04T22:44:49,240 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 9840c0493aab,38659,1733352285296, splitWal=true, meta=true, isMeta: false 2024-12-04T22:44:49,240 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,38659,1733352285296-splitting dir is empty, no logs to split. 2024-12-04T22:44:49,240 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9840c0493aab,38659,1733352285296 WAL count=0, meta=false 2024-12-04T22:44:49,242 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,38659,1733352285296-splitting dir is empty, no logs to split. 2024-12-04T22:44:49,242 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9840c0493aab,38659,1733352285296 WAL count=0, meta=false 2024-12-04T22:44:49,242 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 9840c0493aab,38659,1733352285296 WAL splitting is done? wals=0, meta=false 2024-12-04T22:44:49,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733352286804, region=2d5f9af5ebb9b68bbf2f5ba472b0910b, ASSIGN}] 2024-12-04T22:44:49,245 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733352286804, region=2d5f9af5ebb9b68bbf2f5ba472b0910b, ASSIGN 2024-12-04T22:44:49,246 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733352286804, region=2d5f9af5ebb9b68bbf2f5ba472b0910b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-04T22:44:49,316 INFO [RS:2;9840c0493aab:35557 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T22:44:49,317 INFO [RS:2;9840c0493aab:35557 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9840c0493aab%2C35557%2C1733352289015, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,35557,1733352289015, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs, maxLogs=55 2024-12-04T22:44:49,339 INFO [RS:2;9840c0493aab:35557 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/WALs/9840c0493aab,35557,1733352289015/9840c0493aab%2C35557%2C1733352289015.1733352289318 2024-12-04T22:44:49,340 DEBUG [RS:2;9840c0493aab:35557 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T22:44:49,397 DEBUG [9840c0493aab:44997 {}] balancer.BalancerClusterState(204): Hosts are {9840c0493aab=0} racks are {/default-rack=0} 2024-12-04T22:44:49,397 DEBUG [9840c0493aab:44997 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T22:44:49,397 DEBUG [9840c0493aab:44997 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T22:44:49,397 DEBUG [9840c0493aab:44997 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T22:44:49,398 DEBUG [9840c0493aab:44997 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T22:44:49,398 INFO [9840c0493aab:44997 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T22:44:49,398 INFO [9840c0493aab:44997 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T22:44:49,398 DEBUG [9840c0493aab:44997 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T22:44:49,398 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=2d5f9af5ebb9b68bbf2f5ba472b0910b, regionState=OPENING, regionLocation=9840c0493aab,33479,1733352285233 2024-12-04T22:44:49,400 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9840c0493aab:38659 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9840c0493aab/172.17.0.2:38659 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T22:44:49,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9840c0493aab,38659,1733352285296, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9840c0493aab,38659,1733352285296, seqNum=-1, error=java.net.ConnectException: Call to address=9840c0493aab:38659 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9840c0493aab/172.17.0.2:38659 2024-12-04T22:44:49,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9840c0493aab,38659,1733352285296, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-04T22:44:49,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9840c0493aab,38659,1733352285296, seqNum=-1 from cache 2024-12-04T22:44:49,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 9840c0493aab:38659 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9840c0493aab/172.17.0.2:38659 2024-12-04T22:44:49,510 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T22:44:49,510 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9840c0493aab,33479,1733352285233, seqNum=-1] 2024-12-04T22:44:49,511 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T22:44:49,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36839, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T22:44:49,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733352286804, region=2d5f9af5ebb9b68bbf2f5ba472b0910b, ASSIGN because future has completed 2024-12-04T22:44:49,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d5f9af5ebb9b68bbf2f5ba472b0910b, server=9840c0493aab,33479,1733352285233}] 2024-12-04T22:44:49,672 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:49,673 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d5f9af5ebb9b68bbf2f5ba472b0910b, NAME => 'testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b.', STARTKEY => '', ENDKEY => ''} 2024-12-04T22:44:49,674 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733352286804 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,674 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T22:44:49,674 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,674 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,676 INFO [StoreOpener-2d5f9af5ebb9b68bbf2f5ba472b0910b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,677 INFO [StoreOpener-2d5f9af5ebb9b68bbf2f5ba472b0910b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d5f9af5ebb9b68bbf2f5ba472b0910b columnFamilyName family 2024-12-04T22:44:49,677 DEBUG [StoreOpener-2d5f9af5ebb9b68bbf2f5ba472b0910b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T22:44:49,679 DEBUG [StoreFileOpener-2d5f9af5ebb9b68bbf2f5ba472b0910b-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa, delay=919 ms 2024-12-04T22:44:49,681 DEBUG [StoreOpener-2d5f9af5ebb9b68bbf2f5ba472b0910b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa 2024-12-04T22:44:49,682 INFO [StoreOpener-2d5f9af5ebb9b68bbf2f5ba472b0910b-1 {}] regionserver.HStore(327): Store=2d5f9af5ebb9b68bbf2f5ba472b0910b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T22:44:49,682 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,683 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,684 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,684 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,684 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,686 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,687 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 2d5f9af5ebb9b68bbf2f5ba472b0910b; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66968167, jitterRate=-0.0020965486764907837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T22:44:49,687 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,688 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 2d5f9af5ebb9b68bbf2f5ba472b0910b: Running coprocessor pre-open hook at 1733352289675Writing region info on filesystem at 1733352289675Initializing all the Stores at 1733352289675Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733352289675Cleaning up temporary data from old regions at 1733352289684 (+9 ms)Running coprocessor post-open hooks at 1733352289687 (+3 ms)Region opened successfully at 1733352289688 (+1 ms) 2024-12-04T22:44:49,689 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b., pid=11, masterSystemTime=1733352289669 2024-12-04T22:44:49,693 DEBUG [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:49,693 INFO [RS_OPEN_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:49,693 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=2d5f9af5ebb9b68bbf2f5ba472b0910b, regionState=OPEN, openSeqNum=10, regionLocation=9840c0493aab,33479,1733352285233 2024-12-04T22:44:49,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d5f9af5ebb9b68bbf2f5ba472b0910b, server=9840c0493aab,33479,1733352285233 because future has completed 2024-12-04T22:44:49,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-04T22:44:49,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d5f9af5ebb9b68bbf2f5ba472b0910b, server=9840c0493aab,33479,1733352285233 in 183 msec 2024-12-04T22:44:49,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-12-04T22:44:49,704 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 9840c0493aab,38659,1733352285296 after splitting done 2024-12-04T22:44:49,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733352286804, region=2d5f9af5ebb9b68bbf2f5ba472b0910b, ASSIGN in 458 msec 2024-12-04T22:44:49,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 9840c0493aab,38659,1733352285296, splitWal=true, meta=true in 878 msec 2024-12-04T22:44:49,711 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T22:44:49,711 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T22:44:49,711 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T22:44:49,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:49,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:49,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T22:44:49,712 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T22:44:49,712 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=27575548, stopped=false 2024-12-04T22:44:49,712 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9840c0493aab,44997,1733352285056 2024-12-04T22:44:49,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35557-0x101a284df120003, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T22:44:49,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T22:44:49,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T22:44:49,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:49,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:49,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35557-0x101a284df120003, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:49,762 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T22:44:49,763 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T22:44:49,763 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T22:44:49,763 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35557-0x101a284df120003, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T22:44:49,763 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T22:44:49,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:49,763 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9840c0493aab,33479,1733352285233' ***** 2024-12-04T22:44:49,763 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T22:44:49,763 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9840c0493aab,35557,1733352289015' ***** 2024-12-04T22:44:49,763 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T22:44:49,764 INFO [RS:2;9840c0493aab:35557 {}] bucket.BucketCache(1738): Disabling cache 2024-12-04T22:44:49,764 INFO [RS:2;9840c0493aab:35557 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.persistence 2024-12-04T22:44:49,764 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-04T22:44:49,764 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T22:44:49,765 INFO [RS:0;9840c0493aab:33479 {}] bucket.BucketCache(1738): Disabling cache 2024-12-04T22:44:49,765 INFO [RS:0;9840c0493aab:33479 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8ad35c46-984d-5623-7b67-c7f254581e7f/bucket.persistence 2024-12-04T22:44:49,765 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-04T22:44:49,765 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-04T22:44:49,765 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-04T22:44:49,765 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-04T22:44:49,764 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-04T22:44:49,766 INFO [RS:0;9840c0493aab:33479 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-04T22:44:49,766 INFO [RS:2;9840c0493aab:35557 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-04T22:44:49,766 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-04T22:44:49,766 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-04T22:44:49,776 DEBUG [RS:0;9840c0493aab:33479 {}] bucket.BucketCache(1487): Thread RS:0;9840c0493aab:33479 started persisting bucket cache to file 2024-12-04T22:44:49,776 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T22:44:49,776 DEBUG [RS:2;9840c0493aab:35557 {}] bucket.BucketCache(1487): Thread RS:2;9840c0493aab:35557 started persisting bucket cache to file 2024-12-04T22:44:49,776 DEBUG [RS:0;9840c0493aab:33479 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T22:44:49,776 DEBUG [RS:0;9840c0493aab:33479 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T22:44:49,776 DEBUG [RS:2;9840c0493aab:35557 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T22:44:49,776 DEBUG [RS:2;9840c0493aab:35557 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T22:44:49,776 INFO [regionserver/9840c0493aab:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T22:44:49,780 DEBUG [RS:0;9840c0493aab:33479 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T22:44:49,780 DEBUG [RS:2;9840c0493aab:35557 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T22:44:49,780 DEBUG [RS:2;9840c0493aab:35557 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733352289776 2024-12-04T22:44:49,780 DEBUG [RS:0;9840c0493aab:33479 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733352289776 2024-12-04T22:44:49,780 DEBUG [RS:0;9840c0493aab:33479 {}] bucket.BucketCache(1509): Thread RS:0;9840c0493aab:33479 finished persisting bucket cache to file, renaming 2024-12-04T22:44:49,780 INFO [RS:0;9840c0493aab:33479 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T22:44:49,781 INFO [RS:0;9840c0493aab:33479 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T22:44:49,781 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T22:44:49,781 INFO [RS:0;9840c0493aab:33479 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T22:44:49,781 INFO [regionserver/9840c0493aab:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T22:44:49,781 INFO [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(3091): Received CLOSE for 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,781 INFO [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(959): stopping server 9840c0493aab,33479,1733352285233 2024-12-04T22:44:49,781 INFO [RS:0;9840c0493aab:33479 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T22:44:49,781 INFO [RS:0;9840c0493aab:33479 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9840c0493aab:33479. 2024-12-04T22:44:49,781 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-04T22:44:49,781 DEBUG [RS:0;9840c0493aab:33479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T22:44:49,781 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2d5f9af5ebb9b68bbf2f5ba472b0910b, disabling compactions & flushes 2024-12-04T22:44:49,781 DEBUG [RS:0;9840c0493aab:33479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:49,781 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:49,781 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:49,781 INFO [RS:0;9840c0493aab:33479 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T22:44:49,781 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. after waiting 0 ms 2024-12-04T22:44:49,781 INFO [RS:0;9840c0493aab:33479 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T22:44:49,782 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:49,782 INFO [RS:0;9840c0493aab:33479 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T22:44:49,782 INFO [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T22:44:49,782 INFO [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T22:44:49,782 DEBUG [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(1325): Online Regions={2d5f9af5ebb9b68bbf2f5ba472b0910b=testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T22:44:49,782 DEBUG [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2d5f9af5ebb9b68bbf2f5ba472b0910b 2024-12-04T22:44:49,782 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-04T22:44:49,783 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T22:44:49,783 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T22:44:49,783 DEBUG [StoreFileCloser-2d5f9af5ebb9b68bbf2f5ba472b0910b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/family/1f7ea4a615fb4134a327ddc266c998aa 2024-12-04T22:44:49,783 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T22:44:49,783 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T22:44:49,783 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T22:44:49,783 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-12-04T22:44:49,788 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/default/testBlockEvictionOnGracefulStop1733352286804/2d5f9af5ebb9b68bbf2f5ba472b0910b/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-04T22:44:49,789 INFO [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:49,789 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2d5f9af5ebb9b68bbf2f5ba472b0910b: Waiting for close lock at 1733352289781Running coprocessor pre-close hooks at 1733352289781Disabling compacts and flushes for region at 1733352289781Disabling writes for close at 1733352289781Writing region close event to WAL at 1733352289783 (+2 ms)Running coprocessor post-close hooks at 1733352289789 (+6 ms)Closed at 1733352289789 2024-12-04T22:44:49,789 DEBUG [RS_CLOSE_REGION-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b. 2024-12-04T22:44:49,798 DEBUG [RS:2;9840c0493aab:35557 {}] bucket.BucketCache(1509): Thread RS:2;9840c0493aab:35557 finished persisting bucket cache to file, renaming 2024-12-04T22:44:49,798 WARN [RS:2;9840c0493aab:35557 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-04T22:44:49,798 INFO [RS:2;9840c0493aab:35557 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T22:44:49,798 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c5b8a6cc256f4687839fb78b8843095a 2024-12-04T22:44:49,799 INFO [RS:2;9840c0493aab:35557 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T22:44:49,799 INFO [RS:2;9840c0493aab:35557 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T22:44:49,799 INFO [RS:2;9840c0493aab:35557 {}] regionserver.HRegionServer(959): stopping server 9840c0493aab,35557,1733352289015 2024-12-04T22:44:49,799 INFO [RS:2;9840c0493aab:35557 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T22:44:49,799 INFO [RS:2;9840c0493aab:35557 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;9840c0493aab:35557. 2024-12-04T22:44:49,799 DEBUG [RS:2;9840c0493aab:35557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T22:44:49,799 DEBUG [RS:2;9840c0493aab:35557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:49,799 INFO [RS:2;9840c0493aab:35557 {}] regionserver.HRegionServer(976): stopping server 9840c0493aab,35557,1733352289015; all regions closed. 2024-12-04T22:44:49,806 DEBUG [RS:2;9840c0493aab:35557 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs 2024-12-04T22:44:49,806 INFO [RS:2;9840c0493aab:35557 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9840c0493aab%2C35557%2C1733352289015:(num 1733352289318) 2024-12-04T22:44:49,806 DEBUG [RS:2;9840c0493aab:35557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:49,806 INFO [RS:2;9840c0493aab:35557 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T22:44:49,806 INFO [RS:2;9840c0493aab:35557 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T22:44:49,806 INFO [RS:2;9840c0493aab:35557 {}] hbase.ChoreService(370): Chore service for: regionserver/9840c0493aab:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T22:44:49,807 INFO [RS:2;9840c0493aab:35557 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T22:44:49,807 INFO [RS:2;9840c0493aab:35557 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T22:44:49,807 INFO [regionserver/9840c0493aab:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T22:44:49,807 INFO [RS:2;9840c0493aab:35557 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T22:44:49,807 INFO [RS:2;9840c0493aab:35557 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T22:44:49,807 INFO [RS:2;9840c0493aab:35557 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35557 2024-12-04T22:44:49,819 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/info/c5b8a6cc256f4687839fb78b8843095a is 209, key is testBlockEvictionOnGracefulStop1733352286804,,1733352286809.2d5f9af5ebb9b68bbf2f5ba472b0910b./info:regioninfo/1733352289693/Put/seqid=0 2024-12-04T22:44:49,820 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/info/c5b8a6cc256f4687839fb78b8843095a 2024-12-04T22:44:49,824 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/.tmp/info/c5b8a6cc256f4687839fb78b8843095a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/c5b8a6cc256f4687839fb78b8843095a 2024-12-04T22:44:49,826 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/c5b8a6cc256f4687839fb78b8843095a, delay=989 ms 2024-12-04T22:44:49,828 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/c5b8a6cc256f4687839fb78b8843095a, entries=8, sequenceid=19, filesize=6.6 K 2024-12-04T22:44:49,829 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 46ms, sequenceid=19, compaction requested=false 2024-12-04T22:44:49,830 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/411021807b994048bdb000067667e1cd 2024-12-04T22:44:49,830 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/info/c5b8a6cc256f4687839fb78b8843095a 2024-12-04T22:44:49,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T22:44:49,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35557-0x101a284df120003, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9840c0493aab,35557,1733352289015 2024-12-04T22:44:49,831 INFO [RS:2;9840c0493aab:35557 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T22:44:49,832 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9840c0493aab,35557,1733352289015] 2024-12-04T22:44:49,834 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/ns/f85bf213b7494a738d81ae070bd78aa5 2024-12-04T22:44:49,836 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/table/5029ae9eca094cd9b75fdf92528d8c19 2024-12-04T22:44:49,842 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-12-04T22:44:49,843 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T22:44:49,843 INFO [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T22:44:49,843 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733352289782Running coprocessor pre-close hooks at 1733352289782Disabling compacts and flushes for region at 1733352289782Disabling writes for close at 1733352289783 (+1 ms)Obtaining lock to block concurrent updates at 1733352289783Preparing flush snapshotting stores in 1588230740 at 1733352289783Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1733352289784 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733352289784Flushing 1588230740/info: creating writer at 1733352289784Flushing 1588230740/info: appending metadata at 1733352289819 (+35 ms)Flushing 1588230740/info: closing flushed file at 1733352289819Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29789a50: reopening flushed file at 1733352289823 (+4 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 46ms, sequenceid=19, compaction requested=false at 1733352289829 (+6 ms)Writing region close event to WAL at 1733352289837 (+8 ms)Running coprocessor post-close hooks at 1733352289843 (+6 ms)Closed at 1733352289843 2024-12-04T22:44:49,843 DEBUG [RS_CLOSE_META-regionserver/9840c0493aab:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T22:44:49,851 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9840c0493aab,35557,1733352289015 already deleted, retry=false 2024-12-04T22:44:49,852 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9840c0493aab,35557,1733352289015 expired; onlineServers=1 2024-12-04T22:44:49,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35557-0x101a284df120003, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:49,941 INFO [RS:2;9840c0493aab:35557 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T22:44:49,941 INFO [RS:2;9840c0493aab:35557 {}] regionserver.HRegionServer(1031): Exiting; stopping=9840c0493aab,35557,1733352289015; zookeeper connection closed. 2024-12-04T22:44:49,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35557-0x101a284df120003, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:49,942 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@11c0fcb3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@11c0fcb3 2024-12-04T22:44:49,983 INFO [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(976): stopping server 9840c0493aab,33479,1733352285233; all regions closed. 2024-12-04T22:44:49,985 DEBUG [RS:0;9840c0493aab:33479 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs 2024-12-04T22:44:49,985 INFO [RS:0;9840c0493aab:33479 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9840c0493aab%2C33479%2C1733352285233.meta:.meta(num 1733352289169) 2024-12-04T22:44:49,987 DEBUG [RS:0;9840c0493aab:33479 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/oldWALs 2024-12-04T22:44:49,987 INFO [RS:0;9840c0493aab:33479 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9840c0493aab%2C33479%2C1733352285233:(num 1733352286215) 2024-12-04T22:44:49,987 DEBUG [RS:0;9840c0493aab:33479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T22:44:49,987 INFO [RS:0;9840c0493aab:33479 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T22:44:49,987 INFO [RS:0;9840c0493aab:33479 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T22:44:49,987 INFO [RS:0;9840c0493aab:33479 {}] hbase.ChoreService(370): Chore service for: regionserver/9840c0493aab:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T22:44:49,987 INFO [RS:0;9840c0493aab:33479 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T22:44:49,987 INFO [regionserver/9840c0493aab:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T22:44:49,988 INFO [RS:0;9840c0493aab:33479 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33479 2024-12-04T22:44:49,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T22:44:49,999 INFO [RS:0;9840c0493aab:33479 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T22:44:49,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9840c0493aab,33479,1733352285233 2024-12-04T22:44:50,000 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9840c0493aab,33479,1733352285233] 2024-12-04T22:44:50,020 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9840c0493aab,33479,1733352285233 already deleted, retry=false 2024-12-04T22:44:50,020 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9840c0493aab,33479,1733352285233 expired; onlineServers=0 2024-12-04T22:44:50,020 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9840c0493aab,44997,1733352285056' ***** 2024-12-04T22:44:50,020 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T22:44:50,020 INFO [M:0;9840c0493aab:44997 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T22:44:50,020 INFO [M:0;9840c0493aab:44997 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T22:44:50,021 DEBUG [M:0;9840c0493aab:44997 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T22:44:50,021 DEBUG [M:0;9840c0493aab:44997 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T22:44:50,021 DEBUG [master/9840c0493aab:0:becomeActiveMaster-HFileCleaner.small.0-1733352285860 {}] cleaner.HFileCleaner(306): Exit Thread[master/9840c0493aab:0:becomeActiveMaster-HFileCleaner.small.0-1733352285860,5,FailOnTimeoutGroup] 2024-12-04T22:44:50,021 DEBUG [master/9840c0493aab:0:becomeActiveMaster-HFileCleaner.large.0-1733352285858 {}] cleaner.HFileCleaner(306): Exit Thread[master/9840c0493aab:0:becomeActiveMaster-HFileCleaner.large.0-1733352285858,5,FailOnTimeoutGroup] 2024-12-04T22:44:50,021 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T22:44:50,021 INFO [M:0;9840c0493aab:44997 {}] hbase.ChoreService(370): Chore service for: master/9840c0493aab:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T22:44:50,021 INFO [M:0;9840c0493aab:44997 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T22:44:50,021 DEBUG [M:0;9840c0493aab:44997 {}] master.HMaster(1795): Stopping service threads 2024-12-04T22:44:50,021 INFO [M:0;9840c0493aab:44997 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T22:44:50,021 INFO [M:0;9840c0493aab:44997 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T22:44:50,022 INFO [M:0;9840c0493aab:44997 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T22:44:50,022 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T22:44:50,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T22:44:50,031 DEBUG [M:0;9840c0493aab:44997 {}] zookeeper.ZKUtil(347): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T22:44:50,031 WARN [M:0;9840c0493aab:44997 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T22:44:50,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T22:44:50,031 INFO [M:0;9840c0493aab:44997 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/.lastflushedseqids 2024-12-04T22:44:50,038 INFO [M:0;9840c0493aab:44997 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T22:44:50,038 INFO [M:0;9840c0493aab:44997 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T22:44:50,038 DEBUG [M:0;9840c0493aab:44997 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T22:44:50,038 INFO [M:0;9840c0493aab:44997 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:50,039 DEBUG [M:0;9840c0493aab:44997 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:50,039 DEBUG [M:0;9840c0493aab:44997 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T22:44:50,039 DEBUG [M:0;9840c0493aab:44997 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:50,039 INFO [M:0;9840c0493aab:44997 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-12-04T22:44:50,051 DEBUG [M:0;9840c0493aab:44997 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6f3051fcd6d84f81846f0025fc094230 2024-12-04T22:44:50,070 DEBUG [M:0;9840c0493aab:44997 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6f3051fcd6d84f81846f0025fc094230 is 82, key is hbase:meta,,1/info:regioninfo/1733352289220/Put/seqid=0 2024-12-04T22:44:50,071 INFO [M:0;9840c0493aab:44997 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6f3051fcd6d84f81846f0025fc094230 2024-12-04T22:44:50,081 DEBUG [M:0;9840c0493aab:44997 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f66161a07d5844ad8256ac6d593eb25a 2024-12-04T22:44:50,101 DEBUG [M:0;9840c0493aab:44997 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f66161a07d5844ad8256ac6d593eb25a is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733352287243/Put/seqid=0 2024-12-04T22:44:50,102 INFO [M:0;9840c0493aab:44997 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f66161a07d5844ad8256ac6d593eb25a 2024-12-04T22:44:50,110 INFO [RS:0;9840c0493aab:33479 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T22:44:50,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:50,110 INFO [RS:0;9840c0493aab:33479 {}] regionserver.HRegionServer(1031): Exiting; stopping=9840c0493aab,33479,1733352285233; zookeeper connection closed. 2024-12-04T22:44:50,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33479-0x101a284df120001, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:50,110 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5f79202c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5f79202c 2024-12-04T22:44:50,110 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-04T22:44:50,113 DEBUG [M:0;9840c0493aab:44997 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 157a52b4511a4d49b034377ba2970ee4 2024-12-04T22:44:50,134 DEBUG [M:0;9840c0493aab:44997 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/157a52b4511a4d49b034377ba2970ee4 is 69, key is 9840c0493aab,33479,1733352285233/rs:state/1733352285963/Put/seqid=0 2024-12-04T22:44:50,135 INFO [M:0;9840c0493aab:44997 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/157a52b4511a4d49b034377ba2970ee4 2024-12-04T22:44:50,138 INFO [M:0;9840c0493aab:44997 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 157a52b4511a4d49b034377ba2970ee4 2024-12-04T22:44:50,138 DEBUG [M:0;9840c0493aab:44997 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6f3051fcd6d84f81846f0025fc094230 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6f3051fcd6d84f81846f0025fc094230 2024-12-04T22:44:50,141 INFO [M:0;9840c0493aab:44997 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6f3051fcd6d84f81846f0025fc094230, entries=14, sequenceid=102, filesize=5.9 K 2024-12-04T22:44:50,141 DEBUG [M:0;9840c0493aab:44997 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f66161a07d5844ad8256ac6d593eb25a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f66161a07d5844ad8256ac6d593eb25a 2024-12-04T22:44:50,143 INFO [M:0;9840c0493aab:44997 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f66161a07d5844ad8256ac6d593eb25a, entries=11, sequenceid=102, filesize=6.5 K 2024-12-04T22:44:50,144 DEBUG [M:0;9840c0493aab:44997 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/157a52b4511a4d49b034377ba2970ee4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/157a52b4511a4d49b034377ba2970ee4 2024-12-04T22:44:50,146 INFO [M:0;9840c0493aab:44997 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 157a52b4511a4d49b034377ba2970ee4 2024-12-04T22:44:50,147 INFO [M:0;9840c0493aab:44997 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/762e8027-232f-94a6-cd19-d65d8d6b5bd2/b7172295-7ace-f4e2-cdd7-b1d9064b108b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/157a52b4511a4d49b034377ba2970ee4, entries=3, sequenceid=102, filesize=5.3 K 2024-12-04T22:44:50,149 INFO [M:0;9840c0493aab:44997 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=102, compaction requested=false 2024-12-04T22:44:50,151 INFO [M:0;9840c0493aab:44997 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T22:44:50,151 DEBUG [M:0;9840c0493aab:44997 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733352290038Disabling compacts and flushes for region at 1733352290038Disabling writes for close at 1733352290039 (+1 ms)Obtaining lock to block concurrent updates at 1733352290039Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733352290039Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1733352290039Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733352290040 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733352290040Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733352290070 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733352290070Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733352290074 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733352290101 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733352290101Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733352290105 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733352290133 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733352290133Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72a26c: reopening flushed file at 1733352290138 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38fe9516: reopening flushed file at 1733352290141 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@baeeaf5: reopening flushed file at 1733352290144 (+3 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=102, compaction requested=false at 1733352290149 (+5 ms)Writing region close event to WAL at 1733352290150 (+1 ms)Closed at 1733352290150 2024-12-04T22:44:50,152 INFO [M:0;9840c0493aab:44997 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T22:44:50,152 INFO [M:0;9840c0493aab:44997 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44997 2024-12-04T22:44:50,153 INFO [M:0;9840c0493aab:44997 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T22:44:50,154 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T22:44:50,262 INFO [M:0;9840c0493aab:44997 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T22:44:50,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:50,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x101a284df120000, quorum=127.0.0.1:59666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T22:44:50,266 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T22:44:50,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T22:44:50,268 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T22:44:50,276 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 51) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=261 (was 223) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11031 (was 11159)